Subject: [nfsv4] RFC: CLONE operation

From: Christoph Hellwig <[email-masked]>
Date: Thu, 19 Mar 2015 19:01:16 +0100

I spent some time recently with copy offload in Linux, and as part
of that talked to various people at LSF/MM and Vault in Linux.

One conclusion we came to is that the slow and mean generic copy
from here to there (personified by the COPY machinery) is a too
heavy and not always suitable approach for the other kind of
operation people want to overload on it:  a cheap and atomic clone
from one file to another, personified for example by the btrfs
clone and clone_range ioctls, or the equivalent of a cheap copy on
write clone of a VM image.

So below is a proposal for a CLONE operation, which mostly is a subset
of COPY that is limited to intra-server and synchronous operation,
with the added bit that it is atomic in the sense that changes can't
be seen half-way.

I've also done a prototype implementation based on Anna's simple
Linux copy implementation, which is available here:

	http://git.infradead.org/users/hch/pnfs.git/shortlog/refs/heads/clone
	git://git.infradead.org/users/hch/pnfs.git clone

diff --git a/Makefile b/Makefile
index 502c65f..b878c42 100755
--- a/Makefile
+++ b/Makefile
@@ -302,6 +302,8 @@ AUTOGEN =	\
 		autogen/cb_sequence_res.xml \
 		autogen/cb_wants_cancelled_args.xml \
 		autogen/cb_wants_cancelled_res.xml \
+		autogen/clone_args.xml \
+		autogen/clone_res.xml \
 		autogen/close_args.xml \
 		autogen/close_res.xml \
 		autogen/commit_args.xml \
@@ -461,6 +463,7 @@ IDXMLSRC_BASE = \
 	${DOC_PREFIX}_middle_op_read_plus.xml \
 	${DOC_PREFIX}_middle_op_seek.xml \
 	${DOC_PREFIX}_middle_op_write_same.xml \
+	${DOC_PREFIX}_middle_op_clone.xml \
 	${DOC_PREFIX}_middle_op_end.xml \
 	${DOC_PREFIX}_middle_op_cb_start.xml \
 	${DOC_PREFIX}_middle_op_cb_offload.xml \
diff --git a/dotx.d/Makefile b/dotx.d/Makefile
index 552c7ec..0f18763 100755
--- a/dotx.d/Makefile
+++ b/dotx.d/Makefile
@@ -3,6 +3,8 @@
 FORE_OPS_AS_IS1 = \
 		access_args.x \
 		access_res.x \
+		clone_args.x \
+		clone_res.x \
 		close_args.x \
 		close_res.x \
 		commit_args.x \
diff --git a/dotx.d/clone_args.x b/dotx.d/clone_args.x
new file mode 100644
index 0000000..b88a230
--- /dev/null
+++ b/dotx.d/clone_args.x
@@ -0,0 +1,10 @@
+struct CLONE4args {
+	/* SAVED_FH: source file */
+	/* CURRENT_FH: destination file */
+	stateid4	cl_src_stateid;
+	stateid4	cl_dst_stateid;
+	offset4		cl_src_offset;
+	offset4		cl_dst_offset;
+	length4		cl_count;
+};
+
diff --git a/dotx.d/clone_res.x b/dotx.d/clone_res.x
new file mode 100644
index 0000000..9794536
--- /dev/null
+++ b/dotx.d/clone_res.x
@@ -0,0 +1,3 @@
+struct CLONE4res {
+	nfsstat4	cl_status;
+};
diff --git a/dotx.d/spit_types.sh b/dotx.d/spit_types.sh
index fdd4abf..e6af55d 100755
--- a/dotx.d/spit_types.sh
+++ b/dotx.d/spit_types.sh
@@ -990,6 +990,7 @@ enum nfs_opnum4 {
  OP_READ_PLUS		= 68,
  OP_SEEK		= 69,
  OP_WRITE_SAME		= 70,
+ OP_CLONE		= 71,
  OP_ILLEGAL		= 10044
 };
 EOF
@@ -1113,6 +1114,7 @@ union nfs_argop4 switch (nfs_opnum4 argop) {
  case OP_READ_PLUS:	READ_PLUS4args opread_plus;
  case OP_SEEK:		SEEK4args opseek;
  case OP_WRITE_SAME:	WRITE_SAME4args opwrite_same;
+ case OP_CLONE:		CLONE4args opclone;
 
  /* Operations not new to NFSv4.1 */
  case OP_ILLEGAL:	void;
@@ -1246,6 +1248,7 @@ union nfs_resop4 switch (nfs_opnum4 resop) {
  case OP_READ_PLUS:	READ_PLUS4res opread_plus;
  case OP_SEEK:		SEEK4res opseek;
  case OP_WRITE_SAME:	WRITE_SAME4res opwrite_same;
+ case OP_CLONE:		CLONE4res opclone;
 
  /* Operations not new to NFSv4.1 */
  case OP_ILLEGAL:	ILLEGAL4res opillegal;
diff --git a/nfsv42_middle_errors.xml b/nfsv42_middle_errors.xml
index 72893d4..de4337c 100644
--- a/nfsv42_middle_errors.xml
+++ b/nfsv42_middle_errors.xml
@@ -1413,6 +1413,41 @@
         NFS4ERR_TOO_MANY_OPS
       </c>
 
+      <c>CLONE</c>
+      <c>
+        NFS4ERR_ACCESS,
+        NFS4ERR_ADMIN_REVOKED,
+        NFS4ERR_BADXDR,
+        NFS4ERR_BAD_STATEID,
+        NFS4ERR_DEADSESSION,
+        NFS4ERR_DELAY,
+        NFS4ERR_DELEG_REVOKED,
+        NFS4ERR_EXPIRED,
+        NFS4ERR_FBIG,
+        NFS4ERR_FHEXPIRED,
+        NFS4ERR_GRACE,
+        NFS4ERR_INVAL,
+        NFS4ERR_IO,
+        NFS4ERR_ISDIR,
+        NFS4ERR_MOVED,
+        NFS4ERR_NOFILEHANDLE,
+        NFS4ERR_NOTSUPP,
+        NFS4ERR_NOSPC,
+        NFS4ERR_OLD_STATEID,
+        NFS4ERR_OPENMODE,
+        NFS4ERR_OP_NOT_IN_SESSION,
+        NFS4ERR_REP_TOO_BIG,
+        NFS4ERR_REP_TOO_BIG_TO_CACHE,
+        NFS4ERR_REQ_TOO_BIG,
+        NFS4ERR_RETRY_UNCACHED_REP,
+        NFS4ERR_ROFS,
+        NFS4ERR_SERVERFAULT,
+        NFS4ERR_STALE,
+        NFS4ERR_SYMLINK,
+        NFS4ERR_TOO_MANY_OPS,
+        NFS4ERR_WRONG_TYPE
+      </c>
+
       <c>CLOSE</c>
       <c>
         NFS4ERR_ADMIN_REVOKED,
diff --git a/nfsv42_middle_new_errors.xml b/nfsv42_middle_new_errors.xml
index 1328bcf..41947ec 100644
--- a/nfsv42_middle_new_errors.xml
+++ b/nfsv42_middle_new_errors.xml
@@ -158,6 +158,41 @@
         NFS4ERR_TOO_MANY_OPS,
         NFS4ERR_WRONG_TYPE
       </c>
+ 
+      <c>CLONE</c>
+      <c>
+        NFS4ERR_ACCESS,
+        NFS4ERR_ADMIN_REVOKED,
+        NFS4ERR_BADXDR,
+        NFS4ERR_BAD_STATEID,
+        NFS4ERR_DEADSESSION,
+        NFS4ERR_DELAY,
+        NFS4ERR_DELEG_REVOKED,
+        NFS4ERR_EXPIRED,
+        NFS4ERR_FBIG,
+        NFS4ERR_FHEXPIRED,
+        NFS4ERR_GRACE,
+        NFS4ERR_INVAL,
+        NFS4ERR_IO,
+        NFS4ERR_ISDIR,
+        NFS4ERR_MOVED,
+        NFS4ERR_NOFILEHANDLE,
+        NFS4ERR_NOTSUPP,
+        NFS4ERR_NOSPC,
+        NFS4ERR_OLD_STATEID,
+        NFS4ERR_OPENMODE,
+        NFS4ERR_OP_NOT_IN_SESSION,
+        NFS4ERR_REP_TOO_BIG,
+        NFS4ERR_REP_TOO_BIG_TO_CACHE,
+        NFS4ERR_REQ_TOO_BIG,
+        NFS4ERR_RETRY_UNCACHED_REP,
+        NFS4ERR_ROFS,
+        NFS4ERR_SERVERFAULT,
+        NFS4ERR_STALE,
+        NFS4ERR_SYMLINK,
+        NFS4ERR_TOO_MANY_OPS,
+        NFS4ERR_WRONG_TYPE
+      </c>
 
       <c>COPY</c>
       <c>
diff --git a/nfsv42_middle_op_clone.xml b/nfsv42_middle_op_clone.xml
new file mode 100644
index 0000000..e6ecfd2
--- /dev/null
+++ b/nfsv42_middle_op_clone.xml
@@ -0,0 +1,98 @@
+<!-- Copyright (C) The IETF Trust (2011-2014) -->
+<!-- Copyright (C) The Internet Society (2011-2014) -->
+
+<section anchor="op:clone" title="Operation 71: CLONE - Clone a file">
+  <section toc='exclude' title="ARGUMENT">
+    <t>
+      &lt;CODE BEGINS&gt;
+    </t>
+
+    <?rfc include='autogen/clone_args.xml'?>
+
+    <t>
+      &lt;CODE ENDS&gt;
+    </t>
+  </section>
+
+  <section toc='exclude' title="RESULT">
+    <t>
+      &lt;CODE BEGINS&gt;
+    </t>
+
+    <?rfc include='autogen/clone_res.xml'?>
+
+    <t>
+      &lt;CODE ENDS&gt;
+    </t>
+  </section>
+
+  <section toc='exclude' title="DESCRIPTION">
+    <t>
+      The CLONE operation is used to clone file content from a source
+      file specified by the SAVED_FH value into a destination file
+      specified by CURRENT_FH without actually copying the data, e.g.
+      by using copy on write mechanisms.
+    </t>
+
+    <t>
+      Both SAVED_FH and CURRENT_FH must be a regular files. If
+      either of them is not a regular file, the operation MUST fail
+      and return NFS4ERR_WRONG_TYPE.
+    </t>
+
+    <t>
+      In order to set SAVED_FH to the source file handle, the
+      compound procedure requesting the COPY will include a
+      sub-sequence of operations such as
+    </t>
+
+    <figure>
+      <artwork>
+   PUTFH source-fh
+   SAVEFH
+      </artwork>
+    </figure>
+
+    <t>
+      Both cl_src_stateid and cl_dst_stateid MUST refer to either
+      delegation, locking, or open states provided earlier by the
+      server.  The order of selection is explained in Section 8.2.5
+      of <xref target='RFC5661' />. If a stateid is invalid, then the
+      operation MUST fail.
+    </t>
+
+    <t>
+      The cl_src_offset is the offset within the source file from
+      which the data will be read, the cl_dst_offset is the offset
+      within the destination file to which the data will be written,
+      and the cl_count is the number of bytes that will be cloned.
+      An offset of 0 (zero) specifies the start of the file. A count
+      of 0 (zero) requests that all bytes from ca_src_offset through
+      EOF be cloned to the destination.
+    </t>
+
+    <t>
+      If the source offset or the source offset plus count is greater
+      than or equal to the size of the source file, the operation
+      will fail with NFS4ERR_INVAL. The destination offset or
+      destination offset plus count may be greater than the size
+      of the destination file.
+    </t>
+
+    <t>
+      If the COPY wrote past the end of the file on the destination, then the
+      last byte written to will determine the new file size. The contents of
+      any block not written to and past the original size of the file will be
+      as if a normal WRITE extended the file. 
+    </t>
+
+    <t>
+      The CLONE operation is atomic, that is either all changes or
+      no changes are seen by the client or other clients.
+    </t>
+
+    <t>
+      The completion status of the operation is indicated by cr_status.
+    </t>
+  </section>
+</section>
diff --git a/nfsv42_middle_op_mandlist.xml b/nfsv42_middle_op_mandlist.xml
index fe2a90f..82385c7 100644
--- a/nfsv42_middle_op_mandlist.xml
+++ b/nfsv42_middle_op_mandlist.xml
@@ -117,6 +117,7 @@
       <c>ACCESS               </c> <c> REQ </c> <c>              </c>
       <c>BACKCHANNEL_CTL      </c> <c> REQ </c> <c>              </c>
       <c>BIND_CONN_TO_SESSION </c> <c> REQ </c> <c>              </c>
+      <c>CLONE                </c> <c> OPT </c> <c>              </c>
       <c>CLOSE                </c> <c> REQ </c> <c>              </c>
       <c>COMMIT               </c> <c> REQ </c> <c>              </c>
       <c>COPY                 </c> <c> OPT </c> <c> COPYer (REQ), COPYra (REQ) </c>
-- 
1.9.1

---

From: Benny Halevy <[email-masked]>
Date: Mon, 23 Mar 2015 12:11:07 +0200

Hi Christoph,

On 2015-03-19 20:01, Christoph Hellwig
wrote:

I am not sure there is justification to add yet another operation
for the CLONE case.

There is nothing preventing a server of cloning file contents rather
than copying

in the intra-server COPY offload case, right?  If so, can the server
implementation

be limited to that case only while rejecting more complicated cases
it does not support?

Also, in the inter-server scenario, I can see a case where the src
and dest servers

are clustered and use shared storage such as the copy operation may
turn into a clone

by the back-end implementation and these semantics better be covered
as well by the

COPY operation.

+ <c>CLONE</c>
+ <c>
+ NFS4ERR_ACCESS,
+ NFS4ERR_ADMIN_REVOKED,
+ NFS4ERR_BADXDR,
+ NFS4ERR_BAD_STATEID,
+ NFS4ERR_DEADSESSION,
+ NFS4ERR_DELAY,
+ NFS4ERR_DELEG_REVOKED,
+ NFS4ERR_EXPIRED,
+ NFS4ERR_FBIG,
+ NFS4ERR_FHEXPIRED,
+ NFS4ERR_GRACE,
+ NFS4ERR_INVAL,
+ NFS4ERR_IO,
+ NFS4ERR_ISDIR,
+ NFS4ERR_MOVED,
+ NFS4ERR_NOFILEHANDLE,
+ NFS4ERR_NOTSUPP,
+ NFS4ERR_NOSPC,
+ NFS4ERR_OLD_STATEID,
+ NFS4ERR_OPENMODE,

<c>COPY</c>
<c>

Note: holes may be clones too.  This should lead into deallocating
the respective

blocks at the destination range.

+ </t>
+
+ <t>

s/a //

Also, stating the obvious, cl_dst_stateid MUST refer to a file
OPENed for

OPEN4_SHARE_ACCESS_WRITE or OPEN4_SHARE_ACCESS_BOTH.

+
+ <figure>
+ <artwork>
+ PUTFH source-fh
+ SAVEFH
+ </artwork>
+ </figure>
+
+ <t>

Why not allow special stateids, in particular for cl_src_stateid?

+
+ <t>

What's the problem with the (cl_src_offset + cl_count == size(source
file)) case?

How about misaligned byte ranges? Assuming the server needs to split
blocks for that can it

always assure atomicity?

Another interesting case is intra-file clone, i.e. where the src and
destination filehandles

refer to the same file, *and* the src and dst ranges intersect.  In
this case, IMO,

the spec should advise the server to detect whether the intersecting
range is one these

3 distinct cases:

a. cl_src_offset == cl_dst_offset: In this case, do nothing.

b. cl_src_offset < cl_dst_offset && (cl_src_offset  +
cl_count > cl_dst_offset): Clone the range in descending order,
starting from the blocks with the largest offset within
(cl_src_offset + cl_count).

c. cl_src_offset > cl_dst_offset && (cl_dst_offset  +
cl_count > cl_src_offset): Clone the range in ascending order,
starting from the block covering cl_src_offset.

Benny

+
+ <t>

---

From: Tom Haynes <[email-masked]>
Date: Mon, 23 Mar 2015 11:19:32 -0500

> On Mar 23, 2015, at 5:11 AM, Benny Halevy <[email-masked]> wrote:
> 
> Hi Christoph,
> 
>> On 2015-03-19 20:01, Christoph Hellwig wrote:
> I am not sure there is justification to add yet another operation for the CLONE case.
> There is nothing preventing a server of cloning file contents rather than copying
> in the intra-server COPY offload case, right? 

While that is correct, the difference is that CLONE is always synchronous and COPY can be asynchronous.

I.e., the client has to be prepared to handle the callbacks, etc.

> If so, can the server implementation
> be limited to that case only while rejecting more complicated cases it does not support?
> 

But how does the client know the server will be doing that?

Again, the importance of CLONE is to the client, not the server.

> Also, in the inter-server scenario, I can see a case where the src and dest servers
> are clustered and use shared storage such as the copy operation may turn into a clone
> by the back-end implementation and these semantics better be covered as well by the
> COPY operation.
> 
>> 
> Note: holes may be clones too.  This should lead into deallocating the respective
> Also, stating the obvious, cl_dst_stateid MUST refer to a file OPENed for
> What's the problem with the (cl_src_offset + cl_count == size(source file)) case? 
> 
> How about misaligned byte ranges? Assuming the server needs to split blocks for that can it
> always assure atomicity?
> 
> Another interesting case is intra-file clone, i.e. where the src and destination filehandles
> refer to the same file, *and* the src and dst ranges intersect.  In this case, IMO,
> the spec should advise the server to detect whether the intersecting range is one these
> 3 distinct cases:
> 
> b. cl_src_offset < cl_dst_offset && (cl_src_offset  + cl_count > cl_dst_offset): Clone the range in descending order, starting from the blocks with the largest offset within (cl_src_offset + cl_count).
> c. cl_src_offset > cl_dst_offset && (cl_dst_offset  + cl_count > cl_src_offset): Clone the range in ascending order, starting from the block covering cl_src_offset.
> 
> Benny
> 
>> +
>> +    <t>

---

From: Benny Halevy <[email-masked]>
Date: Mon, 23 Mar 2015 18:34:25 +0200

On 2015-03-23 12:11, Benny Halevy
wrote:

Hi Christoph,

On 2015-03-19 20:01, Christoph
Hellwig wrote:

There is nothing preventing a server of cloning file contents
rather than copying

in the intra-server COPY offload case, right?  If so, can the
server implementation

be limited to that case only while rejecting more complicated
cases it does not support?

are clustered and use shared storage such as the copy operation
may turn into a clone

by the back-end implementation and these semantics better be
covered as well by the

COPY operation.

+ <c>CLONE</c>
+ <c>
+ NFS4ERR_ACCESS,
+ NFS4ERR_ADMIN_REVOKED,
+ NFS4ERR_BADXDR,
+ NFS4ERR_BAD_STATEID,
+ NFS4ERR_DEADSESSION,
+ NFS4ERR_DELAY,
+ NFS4ERR_DELEG_REVOKED,
+ NFS4ERR_EXPIRED,
+ NFS4ERR_FBIG,
+ NFS4ERR_FHEXPIRED,
+ NFS4ERR_GRACE,
+ NFS4ERR_INVAL,
+ NFS4ERR_IO,
+ NFS4ERR_ISDIR,
+ NFS4ERR_MOVED,
+ NFS4ERR_NOFILEHANDLE,
+ NFS4ERR_NOTSUPP,
+ NFS4ERR_NOSPC,
+ NFS4ERR_OLD_STATEID,
+ NFS4ERR_OPENMODE,

I'd also add NFS4ERR_XDEV for servers that can't clone across fs
boundaries.

Benny

+ </c>
+
<c>CLOSE</c>
<c>
NFS4ERR_ADMIN_REVOKED,

<c>COPY</c>
<c>

+ </t>
+
+ <t>

s/a //

+
+ <figure>
+ <artwork>
+ PUTFH source-fh
+ SAVEFH
+ </artwork>
+ </figure>
+
+ <t>

+
+ <t>

What's the problem with the (cl_src_offset + cl_count ==
size(source file)) case?

How about misaligned byte ranges? Assuming the server needs to
split blocks for that can it

always assure atomicity?

Another interesting case is intra-file clone, i.e. where the src
and destination filehandles

refer to the same file, *and* the src and dst ranges intersect.
In this case, IMO,

the spec should advise the server to detect whether the
intersecting range is one these

3 distinct cases:

Benny

+
+ <t>

---

From: Marc Eshel <[email-masked]>
Date: Mon, 23 Mar 2015 12:08:34 -0700

I also think that CLONE should be a separate operation, it is mus simpler 
and should not be part of COPY. I might want to implement CLONE but not 
COPY.

Marc.

From:   Tom Haynes <[email-masked]>
To:     Benny Halevy <[email-masked]>
Cc:     Christoph Hellwig <[email-masked]>, "[email-masked]" <[email-masked]>
Date:   03/23/2015 09:21 AM
Subject:        Re: [nfsv4] RFC: CLONE operation
Sent by:        "nfsv4" <[email-masked]>

Hi Christoph,

I am not sure there is justification to add yet another operation for the 
CLONE case.
There is nothing preventing a server of cloning file contents rather than 
copying
in the intra-server COPY offload case, right? 

While that is correct, the difference is that CLONE is always synchronous 
and COPY can be asynchronous.

If so, can the server implementation
be limited to that case only while rejecting more complicated cases it 
does not support?

Also, in the inter-server scenario, I can see a case where the src and 
dest servers
are clustered and use shared storage such as the copy operation may turn 
into a clone
by the back-end implementation and these semantics better be covered as 
well by the
COPY operation.

 

+      <c>CLONE</c>
+      <c>
+        NFS4ERR_ACCESS,
+        NFS4ERR_ADMIN_REVOKED,
+        NFS4ERR_BADXDR,
+        NFS4ERR_BAD_STATEID,
+        NFS4ERR_DEADSESSION,
+        NFS4ERR_DELAY,
+        NFS4ERR_DELEG_REVOKED,
+        NFS4ERR_EXPIRED,
+        NFS4ERR_FBIG,
+        NFS4ERR_FHEXPIRED,
+        NFS4ERR_GRACE,
+        NFS4ERR_INVAL,
+        NFS4ERR_IO,
+        NFS4ERR_ISDIR,
+        NFS4ERR_MOVED,
+        NFS4ERR_NOFILEHANDLE,
+        NFS4ERR_NOTSUPP,
+        NFS4ERR_NOSPC,
+        NFS4ERR_OLD_STATEID,
+        NFS4ERR_OPENMODE,
 
       <c>COPY</c>
       <c>

Note: holes may be clones too.  This should lead into deallocating the 
respective
blocks at the destination range. 

+    </t>
+
+    <t>

s/a //

+
+    <t>

+      destination offset plus count may be greater than the size
+      of the destination file.
+    </t>
+
+    <t>
+      If the COPY wrote past the end of the file on the destination, then 
the
+      last byte written to will determine the new file size. The contents 
of
+      any block not written to and past the original size of the file 
will be
+      as if a normal WRITE extended the file. 
+    </t>
+
+    <t>
+      The CLONE operation is atomic, that is either all changes or
+      no changes are seen by the client or other clients.
+    </t>

Another interesting case is intra-file clone, i.e. where the src and 
destination filehandles
refer to the same file, *and* the src and dst ranges intersect.  In this 
case, IMO,
the spec should advise the server to detect whether the intersecting range 
is one these
3 distinct cases:

a. cl_src_offset == cl_dst_offset: In this case, do nothing.
b. cl_src_offset < cl_dst_offset && (cl_src_offset  + cl_count > 
cl_dst_offset): Clone the range in descending order, starting from the 
blocks with the largest offset within (cl_src_offset + cl_count).
c. cl_src_offset > cl_dst_offset && (cl_dst_offset  + cl_count > 
cl_src_offset): Clone the range in ascending order, starting from the 
block covering cl_src_offset.

Benny

+
+    <t>
       <c>COPY                 </c> <c> OPT </c> <c> COPYer (REQ), COPYra 
(REQ) </c>

_______________________________________________
nfsv4 mailing list
[email-masked]
https://www.ietf.org/mailman/listinfo/nfsv4

---

From: Christoph Hellwig <[email-masked]>
Date: Mon, 23 Mar 2015 20:24:34 +0100

[I've tried to fix you quote, seems like your mailer turned it into a
html-ized fullquote]

On Mon, Mar 23, 2015 at 12:11:07PM +0200, Benny Halevy wrote:
> copying in the intra-server COPY offload case, right?

No, but that's not the point.  The point is to

 a) avoid the need to implement all the COPY complexityfor servers that
    just want to support a simple clone
 b) to offer an atomic clone, which COPY can't provide

> If so, can the server implementation be limited to that case only while 
> rejecting more complicated cases it does not support?

There is no way to do an atomic clone with COPY as-is, and there is
a lot of overhead required to implement it on the server (and client for that
matter).

>     the respective<br>
>     blocks at the destination range. <br>

It does, but do we need to specify that?  Currently we ignore the
preallocation status for COPY, and CLONE would be the same.  For CLONE
I defintively feel save to say any affect of allocate will be reserved,
but for COPY the result may be implementation defined.

>     </blockquote>
>     <br>
>     OPENed for<br>

Indeed.

>     </blockquote>
>     <br>
>     Why not allow special stateids, in particular for cl_src_stateid?<br>
>     <br>

No good reason, except that this was copied from the language I fixed
up for COPY.  We should probably fix it for both.

> +    &lt;t&gt;
> +      will fail with NFS4ERR_INVAL. The destination offset or</pre>
>     </blockquote>
>     <br>
>     file)) case? <br>

It realy should be greater rather than greater tha or equal here, copy and
paste error from COPY.

>     blocks for that can it<br>
>     always assure atomicity?<br>

Any server implementation I can image would reject this.  Thanks for the
remainer, I think we need to add a clause that the clone needs to be aligned.
Unfortunately it seems like we only export a layout_blksize if pNFS is
enabled, but not for the regular NFS case.

>     <br>
>     destination filehandles<br>
>     this case, IMO,<br>
>     range is one these<br>
>     3 distinct cases:<br>

I would prefer to not allow this case for either COPY or CLONE, or does
someone have a good use case for it?  All the local CLONE equivalents
I know of don't allow it.

---

From: Tom Haynes <[email-masked]>
Date: Tue, 24 Mar 2015 09:42:13 -0500

Christoph,

Could you please provide new patches with the fixes you suggest for
both COPY and CLONE?

I’ve heard enough rough consensus here and at the BOF session at
LSF (which included people doing copy offload in different protocols)
to include this into the draft and take the hit for yet another last call.

And btw, if anyone else has an opinion either way, please speak up.

Thanks,
Tom

> On Mar 23, 2015, at 2:24 PM, Christoph Hellwig <[email-masked]> wrote:
>

---

From: [email-masked] (J. Bruce Fields)
Date: Tue, 24 Mar 2015 11:35:54 -0400

On Mon, Mar 23, 2015 at 08:24:34PM +0100, Christoph Hellwig wrote:
...
> On Mon, Mar 23, 2015 at 12:11:07PM +0200, Benny Halevy wrote:
...
> enabled, but not for the regular NFS case.

Is it really important to be able to do ranges, or should we leave that
to a later CLONE_RANGE call if someone needs it?

Are there filesystem implementations that will have limitations other
than layout_blksize-alignment?

--b.

---

From: "Adamson, Andy" <[email-masked]>
Date: Tue, 24 Mar 2015 15:41:47 +0000

> On Mar 24, 2015, at 10:42 AM, Tom Haynes <[email-masked]> wrote:
> 
> Christoph,
> 

A couple of questions.

1) How does the NFS client chooses between the sync/async COPY (both intra and inter cases) ?

2) When is a non-atomic intra server synch COPY better than an atomic intra-server sync CLONE?

—>Andy

---

From: [email-masked] (J. Bruce Fields)
Date: Tue, 24 Mar 2015 11:52:21 -0400

> And btw, if anyone else has an opinion either way, please speak up.

I think both CLONE and COPY are important and we should get them to
users as soon as we can, but they're also both in the middle of active
discussion and prototyping and I don't see the point of tying our hands
before we're sure of the implementation.

Therefore I think we should keep both CLONE and COPY out of
draft-ietf-nfsv4-minorversion2 and instead add them to 4.2 using
something like the process described in

	http://tools.ietf.org/html/draft-ietf-nfsv4-versioning-00

It'd decrease the chance of another delay for
draft-ietf-nfsv4-minorversion2, and it shouldn't significantly delay
getting CLONE or COPY to users unless there's something wrong with the
proposed minor-versioning-extension process.

--b.

---

From: Marc Eshel <[email-masked]>
Date: Tue, 24 Mar 2015 10:00:02 -0700

"nfsv4" <[email-masked]> wrote on 03/24/2015 08:52:21 AM:

> From: [email-masked] (J. Bruce Fields)
> To: Tom Haynes <[email-masked]>
> Cc: Christoph Hellwig <[email-masked]>, [email-masked]
> Date: 03/24/2015 09:01 AM

I agree with Bruce, at least with regards to CLONE, I just recently heard 
of the addition of CLONE, and I would like to experiment with its 
implementation before it in the final spec. It will be a good test for how 
fast we can use the new versioning process.
Marc.

> > >> There is nothing preventing a server of cloning file contents 
rather than
> > > a) avoid the need to implement all the COPY complexity?for servers 
that
> > >>    What's the problem with the (cl_src_offset + cl_count == 
size(source
> > >>    How about misaligned byte ranges? Assuming the server needs to 
split
> > >>    Another interesting case is intra-file clone, i.e. where the src 
and
> > >>    refer to the same file, *and* the src and dst ranges intersect. 
In
> > >>    the spec should advise the server to detect whether the 
intersecting

---

From: Chuck Lever <[email-masked]>
Date: Tue, 24 Mar 2015 13:18:01 -0400

Hi-

It might be constructive if someone (Tom, as author/editor of
the v4.2 spec?) could present a few slides on Thursday regarding
the motivation for CLONE. It should be a brief enough topic.

No controversy, just bring us all up to speed?

--
Chuck Lever
chuck[dot]lever[at]oracle[dot]com

---

From: David Noveck <[email-masked]>
Date: Tue, 24 Mar 2015 13:29:23 -0400

As far as the new versioning process, I think people like it, but there is
still a ways to go before it can be published as an
RFC and we can treat NFSv4.2 as an extensible minor version.    We still
have to go through the process of integrating the minor
versioning rules with the newer process elements and then there's last
call, yet-another-last-call,  final-last-call, and IESG
review.

If people want to go forward right now in the spirit of the newer process,
the most reasonable way to do that would be to produce an individual
draft for CLONE (or a draft for CLONE and COPY, or separate drafts for
CLONE and COPY), depending on what people want to do.
We could then make that/those draft(s) into WG document(s)and do the needed
implementation work in parallel with advancing the
version management work.  I don't think you'd be able to advance these
feature drafts to Proposed Standard until the version
management work was a Proposed Standard, but we could be in a position in
which a lot of implementation had already been done at
that point.

I also like the opportunity this would provide to try out the new process
before we put this before the IESG.  It's always good to
have a good response to the question "How do you know this new process is
going to work?".  "Because it already has"  is a
pretty convincing response.

---

From: Spencer Shepler <[email-masked]>
Date: Tue, 24 Mar 2015 18:00:04 +0000

From: nfsv4 [mailto:[email-masked]] On Behalf Of David Noveck
Sent: Tuesday, March 24, 2015 10:29 AM
To: Marc Eshel
Cc: J. Bruce Fields; nfsv4; [email-masked]; Christoph Hellwig
Subject: Re: [nfsv4] RFC: CLONE operation

As far as the new versioning process, I think people like it, but there is still a ways to go before it can be published as an
RFC and we can treat NFSv4.2 as an extensible minor version.    We still have to go through the process of integrating the minor
versioning rules with the newer process elements and then there's last call, yet-another-last-call,  final-last-call, and IESG
review.

If people want to go forward right now in the spirit of the newer process, the most reasonable way to do that would be to produce an individual
draft for CLONE (or a draft for CLONE and COPY, or separate drafts for CLONE and COPY), depending on what people want to do.
We could then make that/those draft(s) into WG document(s)and do the needed implementation work in parallel with advancing the
version management work.  I don't think you'd be able to advance these feature drafts to Proposed Standard until the version
management work was a Proposed Standard, but we could be in a position in which a lot of implementation had already been done at
that point.
[Spencer] The two I-Ds could move together through the IETF process in that the dependency of one on the other is okay and may help the versioning work.  What I mean by that is it is likely best to have a use of the versioning to demonstrate its utility and correctness and the delivery of CLONE/COPY in that model would be great.

I also like the opportunity this would provide to try out the new process before we put this before the IESG.  It's always good to
have a good response to the question "How do you know this new process is going to work?".  "Because it already has"  is a
pretty convincing response.
[Spencer] Yes, see above.  However, the WG needs to weight the initial concerns you raise above – making the feature dependent on a new process that has not fully been vetted presents a risk.  A manageable risk but it does exist.

Spencer

On Tue, Mar 24, 2015 at 1:00 PM, Marc Eshel <[email-masked]<mailto:[email-masked]>> wrote:
"nfsv4" <[email-masked]<mailto:[email-masked]>> wrote on 03/24/2015 08:52:21 AM:

> From: [email-masked]<mailto:[email-masked]> (J. Bruce Fields)
> To: Tom Haynes <[email-masked]<mailto:[email-masked]>>
> Cc: Christoph Hellwig <[email-masked]<mailto:[email-masked]>>, [email-masked]<mailto:[email-masked]>
> Sent by: "nfsv4" <[email-masked]<mailto:[email-masked]>>
>

>
> --b.
>
> >
> > Thanks,
> > Tom
> >
> > > On Mar 23, 2015, at 2:24 PM, Christoph Hellwig <[email-masked]<mailto:[email-masked]>> wrote:
> > >
rather than
that
size(source
split
and
In
intersecting
> > [email-masked]<mailto:[email-masked]>
> [email-masked]<mailto:[email-masked]>

_______________________________________________
nfsv4 mailing list
[email-masked]<mailto:[email-masked]>
https://www.ietf.org/mailman/listinfo/nfsv4

---

From: "Adamson, Andy" <[email-masked]>
Date: Tue, 24 Mar 2015 18:00:04 +0000

> On Mar 24, 2015, at 11:41 AM, Adamson, Andy <[email-masked]> wrote:
> 
>> 

In other words, I’m not convinced that we need both an intra-server COPY and CLONE.

—>Andy

---

From: Christoph Hellwig <[email-masked]>
Date: Tue, 24 Mar 2015 20:47:51 +0100

> to a later CLONE_RANGE call if someone needs it?

I've seen a fair amout of uses for ranges. btrfs for example started
out with a plane whole file clone and than had to add ranges.  But at
least for the VM image use case a whole file clone should be just fine.

> than layout_blksize-alignment?

All filesystems I know need an alignment to the filesystem blocksize.
layout_blksize just is the closest approximation that I could find so far.

---

From: Christoph Hellwig <[email-masked]>
Date: Tue, 24 Mar 2015 20:49:35 +0100

> 1) How does the NFS client chooses between the sync/async COPY (both intra and inter cases) ?

The way COPY is written it can't chose, the server decideѕ.

> 2) When is a non-atomic intra server synch COPY better than an atomic intra-server sync CLONE?

I don't think it's ever superior, but for servers that can't do COW cloning
and need to do a real copy implementing (async) COPY is feasible, while
implementing CLONE isn't.

---

From: Christoph Hellwig <[email-masked]>
Date: Tue, 24 Mar 2015 20:50:45 +0100

> something like the process described in

As long as we have the same treatment for CLONE and COPY that sounds fine
to me.  I just want to avoid having to rush COPY into environments where
it doesn't fit.

Also given all the last minute issues we found in COPY defering it might be a
good idea to start with..

---

From: "J. Bruce Fields" <[email-masked]>
Date: Tue, 24 Mar 2015 16:04:22 -0400

On Tue, Mar 24, 2015 at 08:47:51PM +0100, Christoph Hellwig wrote:
> out with a plane whole file clone and than had to add ranges.

OK.  Might be worth documenting those use cases if we have any more
information.

> But at

If there's a possibility that'd be a problem then it'd be easy enough to
add an attribute specifically for clone granularity.

--b.

---

From: David Noveck <[email-masked]>
Date: Tue, 24 Mar 2015 17:37:22 -0400

> add an attribute specifically for clone granularity.

I think we should just create this attribute as part of the CLONE feature.
  The
issue is not that that there would be cases in which this is different from
the layout-granularity.
The real issue is that the server might not support pNFS.  adding a new
optional attribute is
clean but changing semantics of layout_blksize post facto is a version
management nightmare.
You don;t want to allow new features to change the semantics of existing
attributes, even if,
as in this particular case, it seems that the result is likely to be benign.

---

From: Anna Schumaker <[email-masked]>
Date: Wed, 25 Mar 2015 13:12:41 -0400

> 1) How does the NFS client chooses between the sync/async COPY (both intra and inter cases) ?

Zach Brown mentioned possibly adding a flag to the upcoming Linux copy syscall that we might be able to use as a selector here.

---

From: Christoph Hellwig <[email-masked]>
Date: Mon, 30 Mar 2015 20:07:46 +0200

Here is an update of the clone proposal.  It adds the new clone_blksize
attribute, and should also incorporate all the other review feedback.

Signed-off-by: Christoph Hellwig <[email-masked]>

diff --git a/Makefile b/Makefile
index c83df3a..198fb66 100755
--- a/Makefile
+++ b/Makefile
@@ -302,6 +302,8 @@ AUTOGEN =	\
 		autogen/cb_sequence_res.xml \
 		autogen/cb_wants_cancelled_args.xml \
 		autogen/cb_wants_cancelled_res.xml \
+		autogen/clone_args.xml \
+		autogen/clone_res.xml \
 		autogen/close_args.xml \
 		autogen/close_res.xml \
 		autogen/commit_args.xml \
@@ -461,6 +463,7 @@ IDXMLSRC_BASE = \
 	${DOC_PREFIX}_middle_op_read_plus.xml \
 	${DOC_PREFIX}_middle_op_seek.xml \
 	${DOC_PREFIX}_middle_op_write_same.xml \
+	${DOC_PREFIX}_middle_op_clone.xml \
 	${DOC_PREFIX}_middle_op_end.xml \
 	${DOC_PREFIX}_middle_op_cb_start.xml \
 	${DOC_PREFIX}_middle_op_cb_offload.xml \
diff --git a/dotx.d/Makefile b/dotx.d/Makefile
index 552c7ec..0f18763 100755
--- a/dotx.d/Makefile
+++ b/dotx.d/Makefile
@@ -3,6 +3,8 @@
 FORE_OPS_AS_IS1 = \
 		access_args.x \
 		access_res.x \
+		clone_args.x \
+		clone_res.x \
 		close_args.x \
 		close_res.x \
 		commit_args.x \
diff --git a/dotx.d/clone_args.x b/dotx.d/clone_args.x
new file mode 100644
index 0000000..b88a230
--- /dev/null
+++ b/dotx.d/clone_args.x
@@ -0,0 +1,10 @@
+struct CLONE4args {
+	/* SAVED_FH: source file */
+	/* CURRENT_FH: destination file */
+	stateid4	cl_src_stateid;
+	stateid4	cl_dst_stateid;
+	offset4		cl_src_offset;
+	offset4		cl_dst_offset;
+	length4		cl_count;
+};
+
diff --git a/dotx.d/clone_res.x b/dotx.d/clone_res.x
new file mode 100644
index 0000000..9794536
--- /dev/null
+++ b/dotx.d/clone_res.x
@@ -0,0 +1,3 @@
+struct CLONE4res {
+	nfsstat4	cl_status;
+};
diff --git a/dotx.d/head.x b/dotx.d/head.x
index 828f506..85fb741 100644
--- a/dotx.d/head.x
+++ b/dotx.d/head.x
@@ -502,6 +502,7 @@ typedef uint64_t	fattr4_space_freed;
 typedef change_attr_type4
 		fattr4_change_attr_type;
 typedef sec_label4	fattr4_sec_label<>;
+typedef uint32_t	fattr4_clone_blksize;
 
 %/*
 % * REQUIRED Attributes
@@ -599,7 +600,7 @@ const FATTR4_FS_CHARSET_CAP	= 76;
 %/*
 % * new to NFSV4.2
 % */
-/* 77 is currently unused */
+const FATTR4_CLONE_BLKSIZE	= 77;
 const FATTR4_SPACE_FREED	= 78;
 const FATTR4_CHANGE_ATTR_TYPE	= 79;
 const FATTR4_SEC_LABEL		= 80;
diff --git a/dotx.d/spit_types.sh b/dotx.d/spit_types.sh
index fdd4abf..e6af55d 100755
--- a/dotx.d/spit_types.sh
+++ b/dotx.d/spit_types.sh
@@ -990,6 +990,7 @@ enum nfs_opnum4 {
  OP_READ_PLUS		= 68,
  OP_SEEK		= 69,
  OP_WRITE_SAME		= 70,
+ OP_CLONE		= 71,
  OP_ILLEGAL		= 10044
 };
 EOF
@@ -1113,6 +1114,7 @@ union nfs_argop4 switch (nfs_opnum4 argop) {
  case OP_READ_PLUS:	READ_PLUS4args opread_plus;
  case OP_SEEK:		SEEK4args opseek;
  case OP_WRITE_SAME:	WRITE_SAME4args opwrite_same;
+ case OP_CLONE:		CLONE4args opclone;

 
+      <c>CLONE</c>
+      <c>
+        NFS4ERR_ACCESS,
+        NFS4ERR_ADMIN_REVOKED,
+        NFS4ERR_BADXDR,
+        NFS4ERR_BAD_STATEID,
+        NFS4ERR_DEADSESSION,
+        NFS4ERR_DELAY,
+        NFS4ERR_DELEG_REVOKED,
+        NFS4ERR_EXPIRED,
+        NFS4ERR_FBIG,
+        NFS4ERR_FHEXPIRED,
+        NFS4ERR_GRACE,
+        NFS4ERR_INVAL,
+        NFS4ERR_IO,
+        NFS4ERR_ISDIR,
+        NFS4ERR_MOVED,
+        NFS4ERR_NOFILEHANDLE,
+        NFS4ERR_NOTSUPP,
+        NFS4ERR_NOSPC,
+        NFS4ERR_OLD_STATEID,
+        NFS4ERR_OPENMODE,
+        NFS4ERR_OP_NOT_IN_SESSION,
+        NFS4ERR_REP_TOO_BIG,
+        NFS4ERR_REP_TOO_BIG_TO_CACHE,
+        NFS4ERR_REQ_TOO_BIG,
+        NFS4ERR_RETRY_UNCACHED_REP,
+        NFS4ERR_ROFS,
+        NFS4ERR_SERVERFAULT,
+        NFS4ERR_STALE,
+        NFS4ERR_SYMLINK,
+        NFS4ERR_TOO_MANY_OPS,
+        NFS4ERR_WRONG_TYPE
+      </c>
+
       <c>CLOSE</c>
       <c>
         NFS4ERR_ADMIN_REVOKED,
diff --git a/nfsv42_middle_fileattributes.xml b/nfsv42_middle_fileattributes.xml
index 200e58c..4a90f27 100644
--- a/nfsv42_middle_fileattributes.xml
+++ b/nfsv42_middle_fileattributes.xml
@@ -53,6 +53,7 @@
       <ttcol align="left">Data Type</ttcol>
       <ttcol align="left">Acc</ttcol>
       <ttcol align="left">Defined in</ttcol>
+      <c>clone_blksize</c>    <c>77</c> <c>length4</c>            <c>R  </c> <c><xref target="ss:fattr:clone" /></c>
       <c>space_freed</c>      <c>78</c> <c>length4</c>            <c>R  </c> <c><xref target="ss:fattr:sf" /></c>
       <c>change_attr_type</c> <c>79</c> <c>change_attr_type4</c>  <c>R  </c> <c><xref target="ss:fattr:chattr" /></c>
       <c>sec_label</c>        <c>80</c> <c>sec_label4</c>         <c>R W</c> <c><xref target="ss:fattr:sec" /></c>
@@ -60,6 +61,12 @@
   </section>
 
   <section anchor="ss:file_attributes:AD" title="Attribute Definitions">
+    <section toc='exclude' anchor="ss:fattr:clone" title="Attribute 77: clone_blksize">
+      <t>
+        The clone_blksize attribute indicates the granularity of a CLONE operation.
+      </t>
+    </section>
+
     <section toc='exclude' anchor="ss:fattr:sf" title="Attribute 78: space_freed">
       <t>
         space_freed gives the number of bytes freed if the file is deleted.
diff --git a/nfsv42_middle_new_errors.xml b/nfsv42_middle_new_errors.xml
index 1328bcf..41947ec 100644
--- a/nfsv42_middle_new_errors.xml
+++ b/nfsv42_middle_new_errors.xml
@@ -158,6 +158,41 @@
         NFS4ERR_TOO_MANY_OPS,
         NFS4ERR_WRONG_TYPE
       </c>
+ 
+      <c>CLONE</c>
+      <c>
+        NFS4ERR_ACCESS,
+        NFS4ERR_ADMIN_REVOKED,
+        NFS4ERR_BADXDR,
+        NFS4ERR_BAD_STATEID,
+        NFS4ERR_DEADSESSION,
+        NFS4ERR_DELAY,
+        NFS4ERR_DELEG_REVOKED,
+        NFS4ERR_EXPIRED,
+        NFS4ERR_FBIG,
+        NFS4ERR_FHEXPIRED,
+        NFS4ERR_GRACE,
+        NFS4ERR_INVAL,
+        NFS4ERR_IO,
+        NFS4ERR_ISDIR,
+        NFS4ERR_MOVED,
+        NFS4ERR_NOFILEHANDLE,
+        NFS4ERR_NOTSUPP,
+        NFS4ERR_NOSPC,
+        NFS4ERR_OLD_STATEID,
+        NFS4ERR_OPENMODE,
+        NFS4ERR_OP_NOT_IN_SESSION,
+        NFS4ERR_REP_TOO_BIG,
+        NFS4ERR_REP_TOO_BIG_TO_CACHE,
+        NFS4ERR_REQ_TOO_BIG,
+        NFS4ERR_RETRY_UNCACHED_REP,
+        NFS4ERR_ROFS,
+        NFS4ERR_SERVERFAULT,
+        NFS4ERR_STALE,
+        NFS4ERR_SYMLINK,
+        NFS4ERR_TOO_MANY_OPS,
+        NFS4ERR_WRONG_TYPE
+      </c>
 
       <c>COPY</c>
       <c>
diff --git a/nfsv42_middle_op_clone.xml b/nfsv42_middle_op_clone.xml
new file mode 100644
index 0000000..50894d6
--- /dev/null
+++ b/nfsv42_middle_op_clone.xml
@@ -0,0 +1,110 @@
+<!-- Copyright (C) The IETF Trust (2011-2014) -->
+<!-- Copyright (C) The Internet Society (2011-2014) -->
+
+<section anchor="op:clone" title="Operation 71: CLONE - Clone a file">
+  <section toc='exclude' title="ARGUMENT">
+    <t>
+      &lt;CODE BEGINS&gt;
+    </t>
+
+    <?rfc include='autogen/clone_args.xml'?>
+
+    <t>
+      &lt;CODE ENDS&gt;
+    </t>
+  </section>
+
+  <section toc='exclude' title="RESULT">
+    <t>
+      &lt;CODE BEGINS&gt;
+    </t>
+
+    <?rfc include='autogen/clone_res.xml'?>
+
+    <t>
+      &lt;CODE ENDS&gt;
+    </t>
+  </section>
+
+  <section toc='exclude' title="DESCRIPTION">
+    <t>
+      The CLONE operation is used to clone file content from a source
+      file specified by the SAVED_FH value into a destination file
+      specified by CURRENT_FH without actually copying the data, e.g.
+      by using copy on write mechanisms.
+    </t>
+
+    <t>
+      Both SAVED_FH and CURRENT_FH must be regular files. If either
+      SAVED_FH or CURRENT_FH are not regular files, the operation
+      MUST fail and return NFS4ERR_WRONG_TYPE.
+    </t>
+
+    <t>
+      SAVED_FH and CURRENT_FH must be different files.  If SAVED_FH and
+      CURRENT_FH refer to the same file, the operation will fail with
+      NFS4ERR_INVAL.
+    </t>
+
+    <t>
+      In order to set SAVED_FH to the source file handle, the
+      compound procedure requesting the COPY will include a
+      sub-sequence of operations such as
+    </t>
+
+    <figure>
+      <artwork>
+   PUTFH source-fh
+   SAVEFH
+      </artwork>
+    </figure>
+
+    <t>
+      The ca_dst_stateid MUST refer to a stateid that is valid for
+      a WRITE operation and follows the rules for stateids in
+      Sections 8.2.5 and 18.32.3 of <xref target='RFC5661' />.
+      The ca_src_stateid MUST refer to a
+      stateid that is valid for a READ operations and follows the rules
+      for stateids in Sections 8.2.5 and 18.22.3 of
+      <xref target='RFC5661' />. If either stateid is invalid, then the
+      operation MUST fail.
+    </t>
+
+    <t>
+      The cl_src_offset is the offset within the source file from
+      which the data will be read, the cl_dst_offset is the offset
+      within the destination file to which the data will be written,
+      and the cl_count is the number of bytes that will be cloned.
+      An offset of 0 (zero) specifies the start of the file. A count
+      of 0 (zero) requests that all bytes from ca_src_offset through
+      EOF be cloned to the destination. Both cl_src_offset and
+      cl_dst_offset must be aligned to the clone block size
+      <xref target='ss:fattr:clone' />.  cl_count must be aligned to
+      the clone block size, unless cl_src_offset + cl_count are
+      equal to the source file size.
+    </t>
+
+    <t>
+      If the source offset or the source offset plus count is greater
+      than the size of the source file, the operation will fail with
+      NFS4ERR_INVAL. The destination offset or destination offset plus
+      count may be greater than the size of the destination file.
+    </t>
+
+    <t>
+      If the COPY wrote past the end of the file on the destination, then the
+      last byte written to will determine the new file size. The contents of
+      any block not written to and past the original size of the file will be
+      as if a normal WRITE extended the file. 
+    </t>
+
+    <t>
+      The CLONE operation is atomic, that is either all changes or
+      no changes are seen by the client or other clients.
+    </t>
+
+    <t>
+      The completion status of the operation is indicated by cr_status.
+    </t>
+  </section>
+</section>
diff --git a/nfsv42_middle_op_mandlist.xml b/nfsv42_middle_op_mandlist.xml
index fe2a90f..82385c7 100644
--- a/nfsv42_middle_op_mandlist.xml
+++ b/nfsv42_middle_op_mandlist.xml
@@ -117,6 +117,7 @@
       <c>ACCESS               </c> <c> REQ </c> <c>              </c>
       <c>BACKCHANNEL_CTL      </c> <c> REQ </c> <c>              </c>
       <c>BIND_CONN_TO_SESSION </c> <c> REQ </c> <c>              </c>
+      <c>CLONE                </c> <c> OPT </c> <c>              </c>
       <c>CLOSE                </c> <c> REQ </c> <c>              </c>
       <c>COMMIT               </c> <c> REQ </c> <c>              </c>
       <c>COPY                 </c> <c> OPT </c> <c> COPYer (REQ), COPYra (REQ) </c>

---

From: Benny Halevy <[email-masked]>
Date: Tue, 31 Mar 2015 07:07:34 +0300

Hi Christoph, as I mentioned on the previous proposal I suggest to add
NFS4ERR_XDEV to the allowed errors for CLONE to be returned if SAVED_FH and
CURRENT_FH refer to files in different filesystems where the server cannot
clone blocks across filesystems.

Thanks,

Benny
On Mar 30, 2015 9:07 PM, "Christoph Hellwig" <[email-masked]> wrote:

> +      <c>clone_blksize</c>    <c>77</c> <c>length4</c>            <c>R
> </c> <c><xref target="ss:fattr:clone" /></c>
>        <c>space_freed</c>      <c>78</c> <c>length4</c>            <c>R
> </c> <c><xref target="ss:fattr:sf" /></c>
>        <c>change_attr_type</c> <c>79</c> <c>change_attr_type4</c>  <c>R
> </c> <c><xref target="ss:fattr:chattr" /></c>
>        <c>sec_label</c>        <c>80</c> <c>sec_label4</c>         <c>R
> W</c> <c><xref target="ss:fattr:sec" /></c>
> @@ -60,6 +61,12 @@
>    </section>
>

---

From: Christoph Hellwig <[email-masked]>
Date: Tue, 31 Mar 2015 11:15:09 +0200

> clone blocks across filesystems.

Sorry, I didn't manage to read that our of the garbage your mailer turned
your text into.  This sounds reasonable, do you want to send a patch with
the proposed wording?  Note that filesystem might not be the right term
here as the clone scope might be very different on different systems.