Merge pull request #5281 from ipfs/kevina/inline-cids

Add support for inlinling via the id-hash

Author: Stebalien

core/commands/add.go

@@ -19,6 +19,7 @@ import (
 	cmds "gx/ipfs/QmPTfgFTo9PFr1PvPKyKoeMgBvYPh6cX3aDP7DHKVbnCbi/go-ipfs-cmds"
 	mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
 	pb "gx/ipfs/QmPtj12fdwuAqj9sBSTNUxBNu8kCGNp8b3o8yUzMm5GHpq/pb"
+	cidutil "gx/ipfs/QmPyxJ2QS7L5FhGkNYkNcXHGjDhvGHueJ4auqAstFHYxy5/go-cidutil"
 	cmdkit "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit"
 	files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files"
 	offline "gx/ipfs/QmZxjqR9Qgompju73kakSoUj3rbVndAzky3oCDiBNCxPs1/go-ipfs-exchange-offline"
@@ -45,6 +46,8 @@ const (
 	fstoreCacheOptionName = "fscache"
 	cidVersionOptionName  = "cid-version"
 	hashOptionName        = "hash"
+	inlineOptionName      = "inline"
+	inlineLimitOptionName = "inline-limit"
 )
 
 const adderOutChanSize = 8
@@ -121,6 +124,8 @@ You can now check what blocks have been created by:
 		cmdkit.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
 		cmdkit.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. (experimental)"),
 		cmdkit.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"),
+		cmdkit.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"),
+		cmdkit.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
 	},
 	PreRun: func(req *cmds.Request, env cmds.Environment) error {
 		quiet, _ := req.Options[quietOptionName].(bool)
@@ -174,6 +179,8 @@ You can now check what blocks have been created by:
 		fscache, _ := req.Options[fstoreCacheOptionName].(bool)
 		cidVer, cidVerSet := req.Options[cidVersionOptionName].(int)
 		hashFunStr, _ := req.Options[hashOptionName].(string)
+		inline, _ := req.Options[inlineOptionName].(bool)
+		inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
 
 		// The arguments are subject to the following constraints.
 		//
@@ -280,7 +287,14 @@ You can now check what blocks have been created by:
 		fileAdder.Silent = silent
 		fileAdder.RawLeaves = rawblks
 		fileAdder.NoCopy = nocopy
-		fileAdder.CidBuilder = &prefix
+		fileAdder.CidBuilder = prefix
+
+		if inline {
+			fileAdder.CidBuilder = cidutil.InlineBuilder{
+				Builder: fileAdder.CidBuilder,
+				Limit:   inlineLimit,
+			}
+		}
 
 		if hash {
 			md := dagtest.Mock()

test/sharness/t0040-add-and-cat.sh

@@ -590,7 +590,7 @@ test_add_cat_expensive "--cid-version=1" "zdj7WcatQrtuE4WMkS4XsfsMixuQN2po4irkYh
 # encoded with the blake2b-256 hash funtion
 test_add_cat_expensive '--hash=blake2b-256' "zDMZof1kwndounDzQCANUHjiE3zt1mPEgx7RE3JTHoZrRRa79xcv"
 
-test_add_named_pipe " Post http://$API_ADDR/api/v0/add?chunker=size-262144&encoding=json&hash=sha2-256&pin=true&progress=true&recursive=true&stream-channels=true:"
+test_add_named_pipe " Post http://$API_ADDR/api/v0/add?chunker=size-262144&encoding=json&hash=sha2-256&inline-limit=32&pin=true&progress=true&recursive=true&stream-channels=true:"
 
 test_add_pwd_is_symlink
 

test/sharness/t0046-id-hash.sh

@@ -43,6 +43,41 @@ test_expect_success "can still fetch it" '
   test_cmp junk.txt actual
 '
 
+test_expect_success "ipfs add --inline works as expected" '
+  echo $ID_HASH0_CONTENTS > afile &&
+  HASH=$(ipfs add -q --inline afile)
+'
+
+test_expect_success "ipfs add --inline uses id multihash" '
+  MHTYPE=`cid-fmt %h $HASH`
+  echo "mhtype is $MHTYPE"
+  test "$MHTYPE" = id
+'
+
+test_expect_success "ipfs add --inline --raw-leaves works as expected" '
+  echo $ID_HASH0_CONTENTS > afile &&
+  HASH=$(ipfs add -q --inline --raw-leaves afile)
+'
+
+test_expect_success "ipfs add --inline --raw-leaves outputs the correct hash" '
+  echo "$ID_HASH0" = "$HASH" &&
+  test "$ID_HASH0" = "$HASH"
+'
+
+test_expect_success "create 1000 bytes file and get its hash" '
+  random 1000 2 > 1000bytes &&
+  HASH0=$(ipfs add -q --raw-leaves --only-hash 1000bytes)
+'
+
+test_expect_success "ipfs add --inline --raw-leaves works as expected on large file" '
+  HASH=$(ipfs add -q --inline --raw-leaves 1000bytes)
+'
+
+test_expect_success "ipfs add --inline --raw-leaves outputs the correct hash on large file" '
+  echo "$HASH0" = "$HASH" &&
+  test "$HASH0" = "$HASH"
+'
+
 test_expect_success "enable filestore" '
   ipfs config --json Experimental.FilestoreEnabled true
 '