From bff7b71389557235c8573a77b0c030765c85fd1f Mon Sep 17 00:00:00 2001 From: bmcquee Date: Sun, 19 Oct 2014 08:51:07 -0700 Subject: [PATCH 01/67] fix func name HasWriableVolume --- go/topology/topology.go | 2 +- go/weed/weed_server/master_server_handlers.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/topology/topology.go b/go/topology/topology.go index c90e8de0b0..876b7c4180 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -109,7 +109,7 @@ func (t *Topology) NextVolumeId() storage.VolumeId { return next } -func (t *Topology) HasWriableVolume(option *VolumeGrowOption) bool { +func (t *Topology) HasWriteableVolume(option *VolumeGrowOption) bool { vl := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index 93e9e7d9a8..7d54521d9f 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -78,7 +78,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) return } - if !ms.Topo.HasWriableVolume(option) { + if !ms.Topo.HasWriteableVolume(option) { if ms.Topo.FreeSpace() <= 0 { w.WriteHeader(http.StatusNotFound) writeJsonQuiet(w, r, operation.AssignResult{Error: "No free volumes left!"}) @@ -86,7 +86,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) } else { ms.vgLock.Lock() defer ms.vgLock.Unlock() - if !ms.Topo.HasWriableVolume(option) { + if !ms.Topo.HasWriteableVolume(option) { if _, err = ms.vg.AutomaticGrowByType(option, ms.Topo); err != nil { writeJsonQuiet(w, r, operation.AssignResult{Error: "Cannot grow volume group! " + err.Error()}) return From 7ca10d8dcfe937ef91a6cfb261bf4f1909815071 Mon Sep 17 00:00:00 2001 From: bmcquee Date: Sun, 19 Oct 2014 20:03:00 -0700 Subject: [PATCH 02/67] add another logging line verbosity 2 --- go/weed/weed_server/raft_server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go index e41867076b..9a21249a91 100644 --- a/go/weed/weed_server/raft_server.go +++ b/go/weed/weed_server/raft_server.go @@ -44,6 +44,7 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin var err error transporter := raft.NewHTTPTransporter("/cluster", 0) transporter.Transport.MaxIdleConnsPerHost = 1024 + glog.V(1).Infof("Starting RaftServer with IP:%v:", httpAddr) s.raftServer, err = raft.NewServer(s.httpAddr, s.dataDir, transporter, nil, topo, "") if err != nil { From 626b896448473b3ce7f2ec40b644c2bfd0e3c714 Mon Sep 17 00:00:00 2001 From: bmcquee Date: Sun, 19 Oct 2014 23:01:15 -0700 Subject: [PATCH 03/67] change wriable to writeable --- go/weed/weed_server/master_server_handlers_admin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index c9a8020c27..17a09dac72 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -143,7 +143,7 @@ func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r * } } -func (ms *MasterServer) hasWriableVolume(option *topology.VolumeGrowOption) bool { +func (ms *MasterServer) hasWriteableVolume(option *topology.VolumeGrowOption) bool { vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } From 02ae8b98a579dda3dd22d97a48a2890aec84cb62 Mon Sep 17 00:00:00 2001 From: bmcquee Date: Sun, 19 Oct 2014 23:07:58 -0700 Subject: [PATCH 04/67] Revert "change wriable to writeable" This reverts commit 626b896448473b3ce7f2ec40b644c2bfd0e3c714. --- go/weed/weed_server/master_server_handlers_admin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index 17a09dac72..c9a8020c27 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -143,7 +143,7 @@ func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r * } } -func (ms *MasterServer) hasWriteableVolume(option *topology.VolumeGrowOption) bool { +func (ms *MasterServer) hasWriableVolume(option *topology.VolumeGrowOption) bool { vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } From bd664def45925d81dfae9c7edfb244d2367170ca Mon Sep 17 00:00:00 2001 From: bmcquee Date: Sun, 19 Oct 2014 23:08:01 -0700 Subject: [PATCH 05/67] Revert "fix func name HasWriableVolume" This reverts commit bff7b71389557235c8573a77b0c030765c85fd1f. --- go/topology/topology.go | 2 +- go/weed/weed_server/master_server_handlers.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/topology/topology.go b/go/topology/topology.go index 876b7c4180..c90e8de0b0 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -109,7 +109,7 @@ func (t *Topology) NextVolumeId() storage.VolumeId { return next } -func (t *Topology) HasWriteableVolume(option *VolumeGrowOption) bool { +func (t *Topology) HasWriableVolume(option *VolumeGrowOption) bool { vl := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index 7d54521d9f..93e9e7d9a8 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -78,7 +78,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) return } - if !ms.Topo.HasWriteableVolume(option) { + if !ms.Topo.HasWriableVolume(option) { if ms.Topo.FreeSpace() <= 0 { w.WriteHeader(http.StatusNotFound) writeJsonQuiet(w, r, operation.AssignResult{Error: "No free volumes left!"}) @@ -86,7 +86,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) } else { ms.vgLock.Lock() defer ms.vgLock.Unlock() - if !ms.Topo.HasWriteableVolume(option) { + if !ms.Topo.HasWriableVolume(option) { if _, err = ms.vg.AutomaticGrowByType(option, ms.Topo); err != nil { writeJsonQuiet(w, r, operation.AssignResult{Error: "Cannot grow volume group! " + err.Error()}) return From e9a8999f63f79406249debb8023be014e8e12eda Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 21 Oct 2014 01:27:06 -0700 Subject: [PATCH 06/67] print error the correct way. --- go/storage/volume.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/go/storage/volume.go b/go/storage/volume.go index de79e91072..60ee1a0c49 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -2,9 +2,9 @@ package storage import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "errors" "fmt" + "github.com/chrislusf/weed-fs/go/glog" "io" "os" "path" @@ -72,7 +72,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool) error { if e != nil { if !os.IsPermission(e) { - return fmt.Errorf("cannot load Volume Data %s.dat: %s", fileName, e.Error()) + return fmt.Errorf("cannot load Volume Data %s.dat: %v", fileName, e) } } @@ -92,12 +92,12 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool) error { if v.readOnly { glog.V(1).Infoln("open to read file", fileName+".idx") if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDONLY, 0644); e != nil { - return fmt.Errorf("cannot read Volume Index %s.idx: %s", fileName, e.Error()) + return fmt.Errorf("cannot read Volume Index %s.idx: %v", fileName, e) } } else { glog.V(1).Infoln("open to write file", fileName+".idx") if indexFile, e = os.OpenFile(fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); e != nil { - return fmt.Errorf("cannot write Volume Index %s.idx: %s", fileName, e.Error()) + return fmt.Errorf("cannot write Volume Index %s.idx: %v", fileName, e) } } glog.V(0).Infoln("loading file", fileName+".idx", "readonly", v.readOnly) @@ -115,7 +115,7 @@ func (v *Volume) Size() int64 { if e == nil { return stat.Size() } - glog.V(0).Infof("Failed to read file size %s %s", v.dataFile.Name(), e.Error()) + glog.V(0).Infof("Failed to read file size %s %v", v.dataFile.Name(), e) return -1 } func (v *Volume) Close() { @@ -170,6 +170,7 @@ func (v *Volume) write(n *Needle) (size uint32, err error) { } var offset int64 if offset, err = v.dataFile.Seek(0, 2); err != nil { + glog.V(0).Infof("faile to seek the end of file: %v", err) return } @@ -177,21 +178,21 @@ func (v *Volume) write(n *Needle) (size uint32, err error) { if offset%NeedlePaddingSize != 0 { offset = offset + (NeedlePaddingSize - offset%NeedlePaddingSize) if offset, err = v.dataFile.Seek(offset, 0); err != nil { - glog.V(4).Infof("failed to align in datafile %s: %s", v.dataFile.Name(), err.Error()) + glog.V(0).Infof("failed to align in datafile %s: %v", v.dataFile.Name(), err) return } } if size, err = n.Append(v.dataFile, v.Version()); err != nil { if e := v.dataFile.Truncate(offset); e != nil { - err = fmt.Errorf("%s\ncannot truncate %s: %s", err, v.dataFile.Name(), e.Error()) + err = fmt.Errorf("%s\ncannot truncate %s: %v", err, v.dataFile.Name(), e) } return } nv, ok := v.nm.Get(n.Id) if !ok || int64(nv.Offset)*NeedlePaddingSize < offset { if _, err = v.nm.Put(n.Id, uint32(offset/NeedlePaddingSize), n.Size); err != nil { - glog.V(4).Infof("failed to save in needle map %d: %s", n.Id, err.Error()) + glog.V(4).Infof("failed to save in needle map %d: %v", n.Id, err) } } if v.lastModifiedTime < n.LastModified { @@ -292,13 +293,13 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId, offset := int64(SuperBlockSize) n, rest, e := ReadNeedleHeader(v.dataFile, version, offset) if e != nil { - err = fmt.Errorf("cannot read needle header: %s", e) + err = fmt.Errorf("cannot read needle header: %v", e) return } for n != nil { if readNeedleBody { if err = n.ReadNeedleBody(v.dataFile, version, offset+int64(NeedleHeaderSize), rest); err != nil { - err = fmt.Errorf("cannot read needle body: %s", err) + err = fmt.Errorf("cannot read needle body: %v", err) return } } @@ -310,7 +311,7 @@ func ScanVolumeFile(dirname string, collection string, id VolumeId, if err == io.EOF { return nil } - return fmt.Errorf("cannot read needle header: %s", err) + return fmt.Errorf("cannot read needle header: %v", err) } } @@ -360,7 +361,7 @@ func (v *Volume) ensureConvertIdxToCdb(fileName string) (cdbCanRead bool) { defer indexFile.Close() glog.V(0).Infof("converting %s.idx to %s.cdb", fileName, fileName) if e = ConvertIndexToCdb(fileName+".cdb", indexFile); e != nil { - glog.V(0).Infof("error converting %s.idx to %s.cdb: %s", fileName, fileName, e.Error()) + glog.V(0).Infof("error converting %s.idx to %s.cdb: %v", fileName, fileName, e) return false } return true From 670b240a266f23fe9d8e5120c61d32065a1519b3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 21 Oct 2014 01:27:40 -0700 Subject: [PATCH 07/67] Fix help text error. --- go/weed/compact.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/weed/compact.go b/go/weed/compact.go index a99e6c93ee..71c4ea90f5 100644 --- a/go/weed/compact.go +++ b/go/weed/compact.go @@ -12,7 +12,7 @@ func init() { var cmdCompact = &Command{ UsageLine: "compact -dir=/tmp -volumeId=234", - Short: "run weed tool compact on volume file if corrupted", + Short: "run weed tool compact on volume file", Long: `Force an compaction to remove deleted files from volume files. The compacted .dat file is stored as .cpd file. The compacted .idx file is stored as .cpx file. From b5aa2ef6050bc052e73536481893011f55005099 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 21 Oct 2014 01:28:17 -0700 Subject: [PATCH 08/67] Add master bind ip address option. --- go/weed/master.go | 3 ++- go/weed/volume.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go/weed/master.go b/go/weed/master.go index 6617c8ca68..f96ff4c08f 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -29,6 +29,7 @@ var cmdMaster = &Command{ var ( mport = cmdMaster.Flag.Int("port", 9333, "http listen port") masterIp = cmdMaster.Flag.String("ip", "", "master listening ip address, default to listen on all network interfaces") + masterBindIp = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") mPublicIp = cmdMaster.Flag.String("publicIp", "", "peer accessible |") metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data") masterPeers = cmdMaster.Flag.String("peers", "", "other master nodes in comma separated ip:port list") @@ -61,7 +62,7 @@ func runMaster(cmd *Command, args []string) bool { *volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold, masterWhiteList, ) - listeningAddress := *masterIp + ":" + strconv.Itoa(*mport) + listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport) glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress) diff --git a/go/weed/volume.go b/go/weed/volume.go index 17d03f0c5b..ce05fcdf35 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -30,7 +30,7 @@ var ( maxVolumeCounts = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...") ip = cmdVolume.Flag.String("ip", "", "ip or server name") publicIp = cmdVolume.Flag.String("publicIp", "", "Publicly accessible ") - bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") + volumeBindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") masterNode = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location") vpulse = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting") vTimeout = cmdVolume.Flag.Int("idleTimeout", 10, "connection idle seconds") @@ -85,7 +85,7 @@ func runVolume(cmd *Command, args []string) bool { *fixJpgOrientation, ) - listeningAddress := *bindIp + ":" + strconv.Itoa(*vport) + listeningAddress := *volumeBindIp + ":" + strconv.Itoa(*vport) glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", listeningAddress) From b85eb293feda0c4b2c322a3d07d17df566df02a4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 23 Oct 2014 09:55:05 -0700 Subject: [PATCH 09/67] fix doc about submit api --- docs/api.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index f26408f5a5..b67463960d 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -187,10 +187,11 @@ Upload File Directly .. code-block:: bash - curl -F file=@/home/chris/myphoto.jpg http://localhost:8080/submit + curl -F file=@/home/chris/myphoto.jpg http://localhost:9333/submit {"fid":"3,01fbe0dc6f1f38","fileName":"myphoto.jpg","fileUrl":"localhost:8080/3,01fbe0dc6f1f38","size":68231} -This API is a little convenient. The volume server would contact the master to get an file id and store it to the right volume server(not necessarily itself). +This API is just for convenience. The master server would get an file id and store the file to the right volume server. +It is a convenient API and does not support different parameters when assigning file id. (or you can add the support and send a push request.) Delete File *********************************** From 30bcda7136f847b59ac6e6610a40829dc10ec8cd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 25 Oct 2014 18:10:32 -0700 Subject: [PATCH 10/67] fix typo --- go/topology/topology.go | 2 +- go/weed/weed_server/master_server_handlers.go | 4 ++-- go/weed/weed_server/master_server_handlers_admin.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/topology/topology.go b/go/topology/topology.go index c90e8de0b0..a81829bb6f 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -109,7 +109,7 @@ func (t *Topology) NextVolumeId() storage.VolumeId { return next } -func (t *Topology) HasWriableVolume(option *VolumeGrowOption) bool { +func (t *Topology) hasWritableVolume(option *VolumeGrowOption) bool { vl := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index 93e9e7d9a8..be2a7ca7e8 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -78,7 +78,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) return } - if !ms.Topo.HasWriableVolume(option) { + if !ms.Topo.hasWritableVolume(option) { if ms.Topo.FreeSpace() <= 0 { w.WriteHeader(http.StatusNotFound) writeJsonQuiet(w, r, operation.AssignResult{Error: "No free volumes left!"}) @@ -86,7 +86,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) } else { ms.vgLock.Lock() defer ms.vgLock.Unlock() - if !ms.Topo.HasWriableVolume(option) { + if !ms.Topo.hasWritableVolume(option) { if _, err = ms.vg.AutomaticGrowByType(option, ms.Topo); err != nil { writeJsonQuiet(w, r, operation.AssignResult{Error: "Cannot grow volume group! " + err.Error()}) return diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index c9a8020c27..080405e542 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -143,7 +143,7 @@ func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r * } } -func (ms *MasterServer) hasWriableVolume(option *topology.VolumeGrowOption) bool { +func (ms *MasterServer) hasWritableVolume(option *topology.VolumeGrowOption) bool { vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } From f527fc1d5ee975fd29e49bf542d893a2d433a491 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 25 Oct 2014 23:45:31 -0700 Subject: [PATCH 11/67] adjust visibility --- go/topology/topology.go | 2 +- go/weed/weed_server/master_server_handlers.go | 4 ++-- go/weed/weed_server/master_server_handlers_admin.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/topology/topology.go b/go/topology/topology.go index a81829bb6f..cfce0c9a8d 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -109,7 +109,7 @@ func (t *Topology) NextVolumeId() storage.VolumeId { return next } -func (t *Topology) hasWritableVolume(option *VolumeGrowOption) bool { +func (t *Topology) HasWritableVolume(option *VolumeGrowOption) bool { vl := t.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index be2a7ca7e8..d7f1f4ce3c 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -78,7 +78,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) return } - if !ms.Topo.hasWritableVolume(option) { + if !ms.Topo.HasWritableVolume(option) { if ms.Topo.FreeSpace() <= 0 { w.WriteHeader(http.StatusNotFound) writeJsonQuiet(w, r, operation.AssignResult{Error: "No free volumes left!"}) @@ -86,7 +86,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) } else { ms.vgLock.Lock() defer ms.vgLock.Unlock() - if !ms.Topo.hasWritableVolume(option) { + if !ms.Topo.HasWritableVolume(option) { if _, err = ms.vg.AutomaticGrowByType(option, ms.Topo); err != nil { writeJsonQuiet(w, r, operation.AssignResult{Error: "Cannot grow volume group! " + err.Error()}) return diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index 080405e542..1a2c6b8e05 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -143,7 +143,7 @@ func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r * } } -func (ms *MasterServer) hasWritableVolume(option *topology.VolumeGrowOption) bool { +func (ms *MasterServer) HasWritableVolume(option *topology.VolumeGrowOption) bool { vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) return vl.GetActiveVolumeCount(option) > 0 } From a5d6e70299559c10126f6ef248f475152303a79a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 26 Oct 2014 11:25:02 -0700 Subject: [PATCH 12/67] fix commenting error. --- go/weed/export.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/weed/export.go b/go/weed/export.go index 81bc21f6eb..9f33d852fb 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -3,9 +3,9 @@ package main import ( "archive/tar" "bytes" + "fmt" "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/storage" - "fmt" "os" "path" "strconv" @@ -36,7 +36,7 @@ var cmdExport = &Command{ var ( exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "input data directory to store volume data files") exportCollection = cmdExport.Flag.String("collection", "", "the volume collection name") - exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.") + exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.") dest = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout") format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}") tarFh *tar.Writer From 179d36ba0e35e4bdad86988a49828836b649e3df Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 26 Oct 2014 11:34:55 -0700 Subject: [PATCH 13/67] formatting code by: goimports -w=true . --- go/filer/client_operations.go | 6 +++--- go/filer/directory.go | 2 -- go/filer/directory_in_map.go | 3 ++- go/filer/filer.go | 2 -- go/filer/filer_embedded.go | 3 ++- go/filer/files_in_leveldb.go | 1 + go/glog/convenient_api.go | 2 -- go/images/orientation.go | 3 ++- go/images/resizing.go | 3 ++- go/operation/assign_file_id.go | 5 +++-- go/operation/data_struts.go | 2 -- go/operation/delete_content.go | 3 ++- go/operation/list_masters.go | 3 ++- go/operation/lookup.go | 3 ++- go/operation/submit.go | 3 ++- go/operation/system_message_test.go | 3 ++- go/operation/upload_content.go | 3 ++- go/sequence/sequence.go | 2 -- go/stats/disk.go | 2 -- go/stats/disk_notsupported.go | 2 -- go/stats/memory_notsupported.go | 2 -- go/storage/cdb_map.go | 5 +++-- go/storage/cdb_map_test.go | 3 ++- go/storage/compact_map.go | 2 -- go/storage/compact_map_perf_test.go | 5 +++-- go/storage/compress.go | 3 ++- go/storage/crc.go | 3 ++- go/storage/file_id.go | 5 +++-- go/storage/needle.go | 7 ++++--- go/storage/needle_map.go | 5 +++-- go/storage/needle_read_write.go | 5 +++-- go/storage/store.go | 9 +++++---- go/storage/store_vacuum.go | 3 ++- go/storage/volume.go | 3 ++- go/storage/volume_super_block.go | 3 ++- go/storage/volume_vacuum.go | 3 ++- go/storage/volume_version.go | 2 -- go/tools/read_index.go | 3 ++- go/topology/allocate_volume.go | 5 +++-- go/topology/data_center.go | 2 -- go/topology/data_node.go | 3 ++- go/topology/node.go | 5 +++-- go/topology/store_replicate.go | 5 +++-- go/topology/topology.go | 7 ++++--- go/topology/topology_event_handling.go | 5 +++-- go/topology/topology_map.go | 2 -- go/topology/topology_vacuum.go | 7 ++++--- go/topology/volume_growth.go | 5 +++-- go/topology/volume_growth_test.go | 5 +++-- go/topology/volume_layout.go | 5 +++-- go/topology/volume_location_list.go | 2 -- go/util/config.go | 3 ++- go/util/constants.go | 2 -- go/util/file_util.go | 3 ++- go/util/net_timeout.go | 3 ++- go/weed/benchmark.go | 7 ++++--- go/weed/download.go | 5 +++-- go/weed/export.go | 5 +++-- go/weed/filer.go | 7 ++++--- go/weed/fix.go | 5 +++-- go/weed/master.go | 9 +++++---- go/weed/mount.go | 2 -- go/weed/mount_std.go | 7 ++++--- go/weed/server.go | 9 +++++---- go/weed/shell.go | 3 ++- go/weed/signal_handling_notsupported.go | 2 -- go/weed/upload.go | 3 ++- go/weed/version.go | 3 ++- go/weed/volume.go | 7 ++++--- go/weed/volume_test.go | 3 ++- go/weed/weed.go | 3 ++- go/weed/weed_server/common.go | 11 ++++++----- go/weed/weed_server/filer_server.go | 5 +++-- go/weed/weed_server/filer_server_handlers.go | 9 +++++---- go/weed/weed_server/filer_server_handlers_admin.go | 3 ++- go/weed/weed_server/master_server.go | 9 +++++---- go/weed/weed_server/master_server_handlers.go | 7 ++++--- go/weed/weed_server/master_server_handlers_admin.go | 13 +++++++------ go/weed/weed_server/raft_server.go | 9 +++++---- go/weed/weed_server/raft_server_handlers.go | 7 ++++--- go/weed/weed_server/volume_server.go | 5 +++-- go/weed/weed_server/volume_server_handlers.go | 13 +++++++------ go/weed/weed_server/volume_server_handlers_admin.go | 5 +++-- .../weed_server/volume_server_handlers_vacuum.go | 3 ++- 84 files changed, 205 insertions(+), 170 deletions(-) diff --git a/go/filer/client_operations.go b/go/filer/client_operations.go index 0b006289ff..b38368735e 100644 --- a/go/filer/client_operations.go +++ b/go/filer/client_operations.go @@ -1,12 +1,12 @@ package filer -import () - import ( - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "fmt" + + "github.com/chrislusf/weed-fs/go/util" + "net/url" ) diff --git a/go/filer/directory.go b/go/filer/directory.go index 956a2f5049..66d1aeba58 100644 --- a/go/filer/directory.go +++ b/go/filer/directory.go @@ -1,7 +1,5 @@ package filer -import () - type DirectoryId int32 type DirectoryEntry struct { diff --git a/go/filer/directory_in_map.go b/go/filer/directory_in_map.go index 1d88a78be0..ee601066c9 100644 --- a/go/filer/directory_in_map.go +++ b/go/filer/directory_in_map.go @@ -2,7 +2,6 @@ package filer import ( "bufio" - "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "os" @@ -10,6 +9,8 @@ import ( "strconv" "strings" "sync" + + "github.com/chrislusf/weed-fs/go/util" ) var writeLock sync.Mutex //serialize changes to dir.log diff --git a/go/filer/filer.go b/go/filer/filer.go index de877fc1f4..bf4a1cb19c 100644 --- a/go/filer/filer.go +++ b/go/filer/filer.go @@ -1,7 +1,5 @@ package filer -import () - type FileId string //file id on weedfs type FileEntry struct { diff --git a/go/filer/filer_embedded.go b/go/filer/filer_embedded.go index 3d3dac9411..0b9b4e6687 100644 --- a/go/filer/filer_embedded.go +++ b/go/filer/filer_embedded.go @@ -1,11 +1,12 @@ package filer import ( - "github.com/chrislusf/weed-fs/go/operation" "errors" "fmt" "path/filepath" "strings" + + "github.com/chrislusf/weed-fs/go/operation" ) type FilerEmbedded struct { diff --git a/go/filer/files_in_leveldb.go b/go/filer/files_in_leveldb.go index 41fbc74bd0..781bb0e5f3 100644 --- a/go/filer/files_in_leveldb.go +++ b/go/filer/files_in_leveldb.go @@ -2,6 +2,7 @@ package filer import ( "bytes" + "github.com/chrislusf/weed-fs/go/glog" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" diff --git a/go/glog/convenient_api.go b/go/glog/convenient_api.go index 3c378083fe..cb43d60e28 100644 --- a/go/glog/convenient_api.go +++ b/go/glog/convenient_api.go @@ -1,7 +1,5 @@ package glog -import () - /* Copying the original glog because it is missing several convenient methods. 1. remove nano time in log format diff --git a/go/images/orientation.go b/go/images/orientation.go index 41ed3f0afb..4bff893116 100644 --- a/go/images/orientation.go +++ b/go/images/orientation.go @@ -2,11 +2,12 @@ package images import ( "bytes" - "github.com/rwcarlsen/goexif/exif" "image" "image/draw" "image/jpeg" "log" + + "github.com/rwcarlsen/goexif/exif" ) //many code is copied from http://camlistore.org/pkg/images/images.go diff --git a/go/images/resizing.go b/go/images/resizing.go index 08a1e15d25..e9de5f7d7e 100644 --- a/go/images/resizing.go +++ b/go/images/resizing.go @@ -2,11 +2,12 @@ package images import ( "bytes" - "github.com/disintegration/imaging" "image" "image/gif" "image/jpeg" "image/png" + + "github.com/disintegration/imaging" ) func Resized(ext string, data []byte, width, height int) (resized []byte, w int, h int) { diff --git a/go/operation/assign_file_id.go b/go/operation/assign_file_id.go index 4e72ad9390..672bfa99c0 100644 --- a/go/operation/assign_file_id.go +++ b/go/operation/assign_file_id.go @@ -1,12 +1,13 @@ package operation import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" "strconv" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type AssignResult struct { diff --git a/go/operation/data_struts.go b/go/operation/data_struts.go index 09fab05d1a..4980f9913a 100644 --- a/go/operation/data_struts.go +++ b/go/operation/data_struts.go @@ -1,7 +1,5 @@ package operation -import () - type JoinResult struct { VolumeSizeLimit uint64 `json:"VolumeSizeLimit,omitempty"` Error string `json:"error,omitempty"` diff --git a/go/operation/delete_content.go b/go/operation/delete_content.go index 84391b6346..416a852b3f 100644 --- a/go/operation/delete_content.go +++ b/go/operation/delete_content.go @@ -1,12 +1,13 @@ package operation import ( - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" "strings" "sync" + + "github.com/chrislusf/weed-fs/go/util" ) type DeleteResult struct { diff --git a/go/operation/list_masters.go b/go/operation/list_masters.go index 7d46a9ebca..7ada942438 100644 --- a/go/operation/list_masters.go +++ b/go/operation/list_masters.go @@ -1,9 +1,10 @@ package operation import ( + "encoding/json" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/util" - "encoding/json" ) type ClusterStatusResult struct { diff --git a/go/operation/lookup.go b/go/operation/lookup.go index ebf153d270..e6b6658daa 100644 --- a/go/operation/lookup.go +++ b/go/operation/lookup.go @@ -1,7 +1,6 @@ package operation import ( - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" _ "fmt" @@ -9,6 +8,8 @@ import ( "net/url" "strings" "time" + + "github.com/chrislusf/weed-fs/go/util" ) type Location struct { diff --git a/go/operation/submit.go b/go/operation/submit.go index 3e09c2edfa..62db466172 100644 --- a/go/operation/submit.go +++ b/go/operation/submit.go @@ -2,13 +2,14 @@ package operation import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "io" "mime" "os" "path" "strconv" "strings" + + "github.com/chrislusf/weed-fs/go/glog" ) type FilePart struct { diff --git a/go/operation/system_message_test.go b/go/operation/system_message_test.go index 2731d0b2fa..b5624c2588 100644 --- a/go/operation/system_message_test.go +++ b/go/operation/system_message_test.go @@ -1,10 +1,11 @@ package operation import ( - proto "code.google.com/p/goprotobuf/proto" "encoding/json" "log" "testing" + + proto "code.google.com/p/goprotobuf/proto" ) func TestSerialDeserial(t *testing.T) { diff --git a/go/operation/upload_content.go b/go/operation/upload_content.go index 38737702d2..480d76dca6 100644 --- a/go/operation/upload_content.go +++ b/go/operation/upload_content.go @@ -2,7 +2,6 @@ package operation import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "encoding/json" "errors" "fmt" @@ -14,6 +13,8 @@ import ( "net/textproto" "path/filepath" "strings" + + "github.com/chrislusf/weed-fs/go/glog" ) type UploadResult struct { diff --git a/go/sequence/sequence.go b/go/sequence/sequence.go index 5a1bceaaf6..1aa167b6b5 100644 --- a/go/sequence/sequence.go +++ b/go/sequence/sequence.go @@ -1,7 +1,5 @@ package sequence -import () - type Sequencer interface { NextFileId(count int) (uint64, int) SetMax(uint64) diff --git a/go/stats/disk.go b/go/stats/disk.go index d5275e5710..46d8c465ef 100644 --- a/go/stats/disk.go +++ b/go/stats/disk.go @@ -1,7 +1,5 @@ package stats -import () - type DiskStatus struct { Dir string All uint64 diff --git a/go/stats/disk_notsupported.go b/go/stats/disk_notsupported.go index 37f9bc47d9..e380d27ea3 100644 --- a/go/stats/disk_notsupported.go +++ b/go/stats/disk_notsupported.go @@ -2,8 +2,6 @@ package stats -import () - func (disk *DiskStatus) fillInStatus() { return } diff --git a/go/stats/memory_notsupported.go b/go/stats/memory_notsupported.go index 64c3d7c2f0..ba82293640 100644 --- a/go/stats/memory_notsupported.go +++ b/go/stats/memory_notsupported.go @@ -2,8 +2,6 @@ package stats -import () - func (mem *MemStatus) fillInStatus() { return } diff --git a/go/storage/cdb_map.go b/go/storage/cdb_map.go index 1869a563e7..fbb59e9c08 100644 --- a/go/storage/cdb_map.go +++ b/go/storage/cdb_map.go @@ -1,13 +1,14 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "fmt" - "github.com/tgulacsi/go-cdb" "os" "path/filepath" + + "github.com/chrislusf/weed-fs/go/util" + "github.com/tgulacsi/go-cdb" ) // CDB-backed read-only needle map diff --git a/go/storage/cdb_map_test.go b/go/storage/cdb_map_test.go index cff7dfa61c..ed690f44f3 100644 --- a/go/storage/cdb_map_test.go +++ b/go/storage/cdb_map_test.go @@ -1,11 +1,12 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" "math/rand" "os" "runtime" "testing" + + "github.com/chrislusf/weed-fs/go/glog" ) var testIndexFilename string = "../../test/sample.idx" diff --git a/go/storage/compact_map.go b/go/storage/compact_map.go index 9cfc3e8f77..6ac18b0125 100644 --- a/go/storage/compact_map.go +++ b/go/storage/compact_map.go @@ -1,7 +1,5 @@ package storage -import () - type NeedleValue struct { Key Key Offset uint32 `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go index ef43de25bf..f746842259 100644 --- a/go/storage/compact_map_perf_test.go +++ b/go/storage/compact_map_perf_test.go @@ -1,11 +1,12 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "log" "os" "testing" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) func TestMemoryUsage(t *testing.T) { diff --git a/go/storage/compress.go b/go/storage/compress.go index a353c9d3a4..5efc9e1f1f 100644 --- a/go/storage/compress.go +++ b/go/storage/compress.go @@ -2,11 +2,12 @@ package storage import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "compress/flate" "compress/gzip" "io/ioutil" "strings" + + "github.com/chrislusf/weed-fs/go/glog" ) /* diff --git a/go/storage/crc.go b/go/storage/crc.go index 7aa4009592..af25b9e535 100644 --- a/go/storage/crc.go +++ b/go/storage/crc.go @@ -1,9 +1,10 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/util" "fmt" "hash/crc32" + + "github.com/chrislusf/weed-fs/go/util" ) var table = crc32.MakeTable(crc32.Castagnoli) diff --git a/go/storage/file_id.go b/go/storage/file_id.go index ec566826cf..f6e36a98c3 100644 --- a/go/storage/file_id.go +++ b/go/storage/file_id.go @@ -1,11 +1,12 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "encoding/hex" "errors" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type FileId struct { diff --git a/go/storage/needle.go b/go/storage/needle.go index daede321b1..aa3206920e 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -1,9 +1,6 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/images" - "github.com/chrislusf/weed-fs/go/util" "encoding/hex" "errors" "io/ioutil" @@ -13,6 +10,10 @@ import ( "strconv" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/images" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go index dca2e6c5d2..504ca15523 100644 --- a/go/storage/needle_map.go +++ b/go/storage/needle_map.go @@ -1,11 +1,12 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "os" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type NeedleMapper interface { diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index bf452ba371..663b5abbd1 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -1,12 +1,13 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" "errors" "fmt" "io" "os" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/store.go b/go/storage/store.go index e7a9dac94e..80d8a30b81 100644 --- a/go/storage/store.go +++ b/go/storage/store.go @@ -1,10 +1,6 @@ package storage import ( - proto "code.google.com/p/goprotobuf/proto" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "fmt" @@ -12,6 +8,11 @@ import ( "math/rand" "strconv" "strings" + + proto "code.google.com/p/goprotobuf/proto" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/store_vacuum.go b/go/storage/store_vacuum.go index 3527e4f592..209e3b4b31 100644 --- a/go/storage/store_vacuum.go +++ b/go/storage/store_vacuum.go @@ -1,9 +1,10 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" "fmt" "strconv" + + "github.com/chrislusf/weed-fs/go/glog" ) func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) { diff --git a/go/storage/volume.go b/go/storage/volume.go index 60ee1a0c49..5b0a836054 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -4,12 +4,13 @@ import ( "bytes" "errors" "fmt" - "github.com/chrislusf/weed-fs/go/glog" "io" "os" "path" "sync" "time" + + "github.com/chrislusf/weed-fs/go/glog" ) type Volume struct { diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go index a7e86b1c3d..57e0deea9a 100644 --- a/go/storage/volume_super_block.go +++ b/go/storage/volume_super_block.go @@ -1,9 +1,10 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" "fmt" "os" + + "github.com/chrislusf/weed-fs/go/glog" ) const ( diff --git a/go/storage/volume_vacuum.go b/go/storage/volume_vacuum.go index b348434d21..7e026a61d9 100644 --- a/go/storage/volume_vacuum.go +++ b/go/storage/volume_vacuum.go @@ -1,10 +1,11 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/glog" "fmt" "os" "time" + + "github.com/chrislusf/weed-fs/go/glog" ) func (v *Volume) garbageLevel() float64 { diff --git a/go/storage/volume_version.go b/go/storage/volume_version.go index 9702ae904f..2e9f58aa29 100644 --- a/go/storage/volume_version.go +++ b/go/storage/volume_version.go @@ -1,7 +1,5 @@ package storage -import () - type Version uint8 const ( diff --git a/go/tools/read_index.go b/go/tools/read_index.go index b99c5b6b84..1104dc3482 100644 --- a/go/tools/read_index.go +++ b/go/tools/read_index.go @@ -1,11 +1,12 @@ package main import ( - "github.com/chrislusf/weed-fs/go/storage" "flag" "fmt" "log" "os" + + "github.com/chrislusf/weed-fs/go/storage" ) var ( diff --git a/go/topology/allocate_volume.go b/go/topology/allocate_volume.go index 6562e9ac5c..a791b4c1c7 100644 --- a/go/topology/allocate_volume.go +++ b/go/topology/allocate_volume.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" + + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) type AllocateVolumeResult struct { diff --git a/go/topology/data_center.go b/go/topology/data_center.go index ebd07803b2..bcf2dfd318 100644 --- a/go/topology/data_center.go +++ b/go/topology/data_center.go @@ -1,7 +1,5 @@ package topology -import () - type DataCenter struct { NodeImpl } diff --git a/go/topology/data_node.go b/go/topology/data_node.go index c3b90470f2..109bd037fe 100644 --- a/go/topology/data_node.go +++ b/go/topology/data_node.go @@ -1,9 +1,10 @@ package topology import ( + "strconv" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/storage" - "strconv" ) type DataNode struct { diff --git a/go/topology/node.go b/go/topology/node.go index 54118802e5..10955fa728 100644 --- a/go/topology/node.go +++ b/go/topology/node.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "errors" "math/rand" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) type NodeId string diff --git a/go/topology/store_replicate.go b/go/topology/store_replicate.go index 6ea019bd86..0c52f9d309 100644 --- a/go/topology/store_replicate.go +++ b/go/topology/store_replicate.go @@ -2,12 +2,13 @@ package topology import ( "bytes" + "net/http" + "strconv" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/operation" "github.com/chrislusf/weed-fs/go/storage" "github.com/chrislusf/weed-fs/go/util" - "net/http" - "strconv" ) func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.VolumeId, needle *storage.Needle, r *http.Request) (size uint32, errorStatus string) { diff --git a/go/topology/topology.go b/go/topology/topology.go index cfce0c9a8d..eb64d336c2 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -1,14 +1,15 @@ package topology import ( + "errors" + "io/ioutil" + "math/rand" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/operation" "github.com/chrislusf/weed-fs/go/sequence" "github.com/chrislusf/weed-fs/go/storage" - "errors" "github.com/goraft/raft" - "io/ioutil" - "math/rand" ) type Topology struct { diff --git a/go/topology/topology_event_handling.go b/go/topology/topology_event_handling.go index eb44914840..7e36568b6a 100644 --- a/go/topology/topology_event_handling.go +++ b/go/topology/topology_event_handling.go @@ -1,10 +1,11 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "math/rand" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func (t *Topology) StartRefreshWritableVolumes(garbageThreshold string) { diff --git a/go/topology/topology_map.go b/go/topology/topology_map.go index d6400c9884..af95c65367 100644 --- a/go/topology/topology_map.go +++ b/go/topology/topology_map.go @@ -1,7 +1,5 @@ package topology -import () - func (t *Topology) ToMap() interface{} { m := make(map[string]interface{}) m["Max"] = t.GetMaxVolumeCount() diff --git a/go/topology/topology_vacuum.go b/go/topology/topology_vacuum.go index 72846f20bf..97a76026d7 100644 --- a/go/topology/topology_vacuum.go +++ b/go/topology/topology_vacuum.go @@ -1,13 +1,14 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "net/url" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) func batchVacuumVolumeCheck(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList, garbageThreshold string) bool { diff --git a/go/topology/volume_growth.go b/go/topology/volume_growth.go index 2859d3992a..b1f241990b 100644 --- a/go/topology/volume_growth.go +++ b/go/topology/volume_growth.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "fmt" "math/rand" "sync" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) /* diff --git a/go/topology/volume_growth_test.go b/go/topology/volume_growth_test.go index 5581c87ce9..267b360427 100644 --- a/go/topology/volume_growth_test.go +++ b/go/topology/volume_growth_test.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/sequence" - "github.com/chrislusf/weed-fs/go/storage" "encoding/json" "fmt" "testing" + + "github.com/chrislusf/weed-fs/go/sequence" + "github.com/chrislusf/weed-fs/go/storage" ) var topologyLayout = ` diff --git a/go/topology/volume_layout.go b/go/topology/volume_layout.go index 7bb0cf7e31..de72bf8952 100644 --- a/go/topology/volume_layout.go +++ b/go/topology/volume_layout.go @@ -1,11 +1,12 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "errors" "math/rand" "sync" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) // mapping from volume to its locations, inverted from server to volume diff --git a/go/topology/volume_location_list.go b/go/topology/volume_location_list.go index 176f469b93..0116140137 100644 --- a/go/topology/volume_location_list.go +++ b/go/topology/volume_location_list.go @@ -1,7 +1,5 @@ package topology -import () - type VolumeLocationList struct { list []*DataNode } diff --git a/go/util/config.go b/go/util/config.go index 050fd0e64c..4cf1d7c645 100644 --- a/go/util/config.go +++ b/go/util/config.go @@ -10,9 +10,10 @@ package util import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" "encoding/json" "os" + + "github.com/chrislusf/weed-fs/go/glog" ) type Config struct { diff --git a/go/util/constants.go b/go/util/constants.go index db1ca38e5b..c407533a85 100644 --- a/go/util/constants.go +++ b/go/util/constants.go @@ -1,7 +1,5 @@ package util -import () - const ( VERSION = "0.64" ) diff --git a/go/util/file_util.go b/go/util/file_util.go index 412d98458b..30e24f001c 100644 --- a/go/util/file_util.go +++ b/go/util/file_util.go @@ -2,9 +2,10 @@ package util import ( "bufio" - "github.com/chrislusf/weed-fs/go/glog" "errors" "os" + + "github.com/chrislusf/weed-fs/go/glog" ) func TestFolderWritable(folder string) (err error) { diff --git a/go/util/net_timeout.go b/go/util/net_timeout.go index eb80822b5e..f274e48022 100644 --- a/go/util/net_timeout.go +++ b/go/util/net_timeout.go @@ -1,9 +1,10 @@ package util import ( - "github.com/chrislusf/weed-fs/go/stats" "net" "time" + + "github.com/chrislusf/weed-fs/go/stats" ) // Listener wraps a net.Listener, and gives a place to store the timeout diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index fec8472e53..8339913cd4 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -2,9 +2,6 @@ package main import ( "bufio" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "math" @@ -16,6 +13,10 @@ import ( "strings" "sync" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) type BenchmarkOptions struct { diff --git a/go/weed/download.go b/go/weed/download.go index c30d179157..c782654f5e 100644 --- a/go/weed/download.go +++ b/go/weed/download.go @@ -1,14 +1,15 @@ package main import ( - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" "fmt" "io" "io/ioutil" "os" "path" "strings" + + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) var ( diff --git a/go/weed/export.go b/go/weed/export.go index 9f33d852fb..c9cc0e3fe8 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -4,14 +4,15 @@ import ( "archive/tar" "bytes" "fmt" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "os" "path" "strconv" "strings" "text/template" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func init() { diff --git a/go/weed/filer.go b/go/weed/filer.go index 7dbecb4d0c..b3fc9c8788 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -1,13 +1,14 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" "net/http" "os" "strconv" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" ) var ( diff --git a/go/weed/fix.go b/go/weed/fix.go index ad573875a1..9c8026448b 100644 --- a/go/weed/fix.go +++ b/go/weed/fix.go @@ -1,11 +1,12 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "os" "path" "strconv" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func init() { diff --git a/go/weed/master.go b/go/weed/master.go index f96ff4c08f..f88964b6dd 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -1,16 +1,17 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" - "github.com/gorilla/mux" "net/http" "os" "runtime" "strconv" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" + "github.com/gorilla/mux" ) func init() { diff --git a/go/weed/mount.go b/go/weed/mount.go index 0290e0f53c..66e645387c 100644 --- a/go/weed/mount.go +++ b/go/weed/mount.go @@ -1,7 +1,5 @@ package main -import () - type MountOptions struct { filer *string dir *string diff --git a/go/weed/mount_std.go b/go/weed/mount_std.go index e5fc0986c8..808c6c563a 100644 --- a/go/weed/mount_std.go +++ b/go/weed/mount_std.go @@ -3,15 +3,16 @@ package main import ( + "fmt" + "os" + "runtime" + "bazil.org/fuse" "bazil.org/fuse/fs" "github.com/chrislusf/weed-fs/go/filer" "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/storage" "github.com/chrislusf/weed-fs/go/util" - "fmt" - "os" - "runtime" ) func runMount(cmd *Command, args []string) bool { diff --git a/go/weed/server.go b/go/weed/server.go index 1d854d6410..6042c22e77 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -1,10 +1,6 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" - "github.com/gorilla/mux" "net/http" "os" "runtime" @@ -13,6 +9,11 @@ import ( "strings" "sync" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" + "github.com/gorilla/mux" ) type ServerOptions struct { diff --git a/go/weed/shell.go b/go/weed/shell.go index c8043e0dd1..f2c4990eab 100644 --- a/go/weed/shell.go +++ b/go/weed/shell.go @@ -2,9 +2,10 @@ package main import ( "bufio" - "github.com/chrislusf/weed-fs/go/glog" "fmt" "os" + + "github.com/chrislusf/weed-fs/go/glog" ) func init() { diff --git a/go/weed/signal_handling_notsupported.go b/go/weed/signal_handling_notsupported.go index ad4f37b0c9..343cf7de2a 100644 --- a/go/weed/signal_handling_notsupported.go +++ b/go/weed/signal_handling_notsupported.go @@ -2,7 +2,5 @@ package main -import () - func OnInterrupt(fn func()) { } diff --git a/go/weed/upload.go b/go/weed/upload.go index 4eae4d274b..2d67c0bd9b 100644 --- a/go/weed/upload.go +++ b/go/weed/upload.go @@ -1,11 +1,12 @@ package main import ( - "github.com/chrislusf/weed-fs/go/operation" "encoding/json" "fmt" "os" "path/filepath" + + "github.com/chrislusf/weed-fs/go/operation" ) var ( diff --git a/go/weed/version.go b/go/weed/version.go index 63441509e0..8d3a6fed7a 100644 --- a/go/weed/version.go +++ b/go/weed/version.go @@ -1,9 +1,10 @@ package main import ( - "github.com/chrislusf/weed-fs/go/util" "fmt" "runtime" + + "github.com/chrislusf/weed-fs/go/util" ) var cmdVersion = &Command{ diff --git a/go/weed/volume.go b/go/weed/volume.go index ce05fcdf35..212cb4b33a 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -1,15 +1,16 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" "net/http" "os" "runtime" "strconv" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" ) func init() { diff --git a/go/weed/volume_test.go b/go/weed/volume_test.go index 764362a2bc..ef00a8c7cd 100644 --- a/go/weed/volume_test.go +++ b/go/weed/volume_test.go @@ -1,10 +1,11 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" "net/http" "testing" "time" + + "github.com/chrislusf/weed-fs/go/glog" ) func TestXYZ(t *testing.T) { diff --git a/go/weed/weed.go b/go/weed/weed.go index c1f5a72de8..c304b7f35d 100644 --- a/go/weed/weed.go +++ b/go/weed/weed.go @@ -1,7 +1,6 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" "flag" "fmt" "io" @@ -13,6 +12,8 @@ import ( "time" "unicode" "unicode/utf8" + + "github.com/chrislusf/weed-fs/go/glog" ) var IsDebug *bool diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index 816107dc5e..39b0830e33 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -2,11 +2,6 @@ package weed_server import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "fmt" "net" @@ -14,6 +9,12 @@ import ( "path/filepath" "strconv" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) var serverStats *stats.ServerStats diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index 5ff0ed9862..9d6f7c71be 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -1,10 +1,11 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/filer" - "github.com/chrislusf/weed-fs/go/glog" "net/http" "strconv" + + "github.com/chrislusf/weed-fs/go/filer" + "github.com/chrislusf/weed-fs/go/glog" ) type FilerServer struct { diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index e36e7c310b..781d7c0fc7 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -1,12 +1,8 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" - "github.com/syndtr/goleveldb/leveldb" "io" "io/ioutil" "math/rand" @@ -14,6 +10,11 @@ import ( "net/url" "strconv" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" + "github.com/syndtr/goleveldb/leveldb" ) func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/filer_server_handlers_admin.go b/go/weed/weed_server/filer_server_handlers_admin.go index ff52dff24d..5ba12e0b8f 100644 --- a/go/weed/weed_server/filer_server_handlers_admin.go +++ b/go/weed/weed_server/filer_server_handlers_admin.go @@ -1,8 +1,9 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" "net/http" + + "github.com/chrislusf/weed-fs/go/glog" ) /* diff --git a/go/weed/weed_server/master_server.go b/go/weed/weed_server/master_server.go index 401f6cfdb6..d000cb6107 100644 --- a/go/weed/weed_server/master_server.go +++ b/go/weed/weed_server/master_server.go @@ -1,16 +1,17 @@ package weed_server import ( + "net/http" + "net/http/httputil" + "net/url" + "sync" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/sequence" "github.com/chrislusf/weed-fs/go/topology" "github.com/chrislusf/weed-fs/go/util" "github.com/goraft/raft" "github.com/gorilla/mux" - "net/http" - "net/http/httputil" - "net/url" - "sync" ) type MasterServer struct { diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index d7f1f4ce3c..7a7a3b70d9 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -1,12 +1,13 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" "net/http" "strconv" "strings" + + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" ) func (ms *MasterServer) lookupVolumeId(vids []string, collection string) (volumeLocations map[string]operation.LookupResult) { diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index 1a2c6b8e05..d7124e5676 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -1,18 +1,19 @@ package weed_server import ( - proto "code.google.com/p/goprotobuf/proto" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/topology" - "github.com/chrislusf/weed-fs/go/util" "encoding/json" "errors" "io/ioutil" "net/http" "strconv" "strings" + + proto "code.google.com/p/goprotobuf/proto" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/util" ) func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go index e41867076b..0b049ce27c 100644 --- a/go/weed/weed_server/raft_server.go +++ b/go/weed/weed_server/raft_server.go @@ -2,19 +2,20 @@ package weed_server import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/topology" "encoding/json" "errors" "fmt" - "github.com/goraft/raft" - "github.com/gorilla/mux" "io/ioutil" "math/rand" "net/http" "net/url" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/topology" + "github.com/goraft/raft" + "github.com/gorilla/mux" ) type RaftServer struct { diff --git a/go/weed/weed_server/raft_server_handlers.go b/go/weed/weed_server/raft_server_handlers.go index 4d51c07677..b466d9afac 100644 --- a/go/weed/weed_server/raft_server_handlers.go +++ b/go/weed/weed_server/raft_server_handlers.go @@ -1,13 +1,14 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" "encoding/json" - "github.com/goraft/raft" "io/ioutil" "net/http" "strings" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/goraft/raft" ) // Handles incoming RAFT joins. diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go index 2a9085f3b6..0a65fd2f67 100644 --- a/go/weed/weed_server/volume_server.go +++ b/go/weed/weed_server/volume_server.go @@ -1,12 +1,13 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" "math/rand" "net/http" "strconv" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) type VolumeServer struct { diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index ce14f6a870..83f6149410 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -1,12 +1,6 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/images" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/topology" "io" "mime" "mime/multipart" @@ -14,6 +8,13 @@ import ( "strconv" "strings" "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/images" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/topology" ) var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") diff --git a/go/weed/weed_server/volume_server_handlers_admin.go b/go/weed/weed_server/volume_server_handlers_admin.go index 1118c80172..caf4c3be8f 100644 --- a/go/weed/weed_server/volume_server_handlers_admin.go +++ b/go/weed/weed_server/volume_server_handlers_admin.go @@ -1,11 +1,12 @@ package weed_server import ( + "net/http" + "path/filepath" + "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/stats" "github.com/chrislusf/weed-fs/go/util" - "net/http" - "path/filepath" ) func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/volume_server_handlers_vacuum.go b/go/weed/weed_server/volume_server_handlers_vacuum.go index b0600d7990..f115e3b8b1 100644 --- a/go/weed/weed_server/volume_server_handlers_vacuum.go +++ b/go/weed/weed_server/volume_server_handlers_vacuum.go @@ -1,8 +1,9 @@ package weed_server import ( - "github.com/chrislusf/weed-fs/go/glog" "net/http" + + "github.com/chrislusf/weed-fs/go/glog" ) func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) { From 7ce628bf09ed949f1686a581d782cab3664511fd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 27 Oct 2014 01:09:45 -0700 Subject: [PATCH 14/67] Clean raft configurations if "peers" option is set. --- go/weed/weed_server/raft_server.go | 46 ++++++++++++++++-------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go index 0b049ce27c..12148db495 100644 --- a/go/weed/weed_server/raft_server.go +++ b/go/weed/weed_server/raft_server.go @@ -9,6 +9,8 @@ import ( "math/rand" "net/http" "net/url" + "os" + "path" "strings" "time" @@ -46,6 +48,13 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin transporter := raft.NewHTTPTransporter("/cluster", 0) transporter.Transport.MaxIdleConnsPerHost = 1024 + // Clear old cluster configurations if peers are set + if len(s.peers) > 0 { + os.RemoveAll(path.Join(s.dataDir, "conf")) + os.RemoveAll(path.Join(s.dataDir, "log")) + os.RemoveAll(path.Join(s.dataDir, "snapshot")) + } + s.raftServer, err = raft.NewServer(s.httpAddr, s.dataDir, transporter, nil, topo, "") if err != nil { glog.V(0).Infoln(err) @@ -53,35 +62,30 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin } transporter.Install(s.raftServer, s) s.raftServer.SetHeartbeatInterval(1 * time.Second) - s.raftServer.SetElectionTimeout(time.Duration(pulseSeconds) * 1150 * time.Millisecond) + s.raftServer.SetElectionTimeout(time.Duration(pulseSeconds) * 3450 * time.Millisecond) s.raftServer.Start() s.router.HandleFunc("/cluster/join", s.joinHandler).Methods("POST") s.router.HandleFunc("/cluster/status", s.statusHandler).Methods("GET") - // Join to leader if specified. if len(s.peers) > 0 { - if !s.raftServer.IsLogEmpty() { - glog.V(0).Infoln("Starting cluster with existing logs.") - } else { - glog.V(0).Infoln("Joining cluster:", strings.Join(s.peers, ",")) - time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond) - firstJoinError := s.Join(s.peers) - if firstJoinError != nil { - glog.V(0).Infoln("No existing server found. Starting as leader in the new cluster.") - _, err := s.raftServer.Do(&raft.DefaultJoinCommand{ - Name: s.raftServer.Name(), - ConnectionString: "http://" + s.httpAddr, - }) - if err != nil { - glog.V(0).Infoln(err) - return nil - } + // Join to leader if specified. + glog.V(0).Infoln("Joining cluster:", strings.Join(s.peers, ",")) + time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond) + firstJoinError := s.Join(s.peers) + if firstJoinError != nil { + glog.V(0).Infoln("No existing server found. Starting as leader in the new cluster.") + _, err := s.raftServer.Do(&raft.DefaultJoinCommand{ + Name: s.raftServer.Name(), + ConnectionString: "http://" + s.httpAddr, + }) + if err != nil { + glog.V(0).Infoln(err) + return nil } } - - // Initialize the server by joining itself. } else if s.raftServer.IsLogEmpty() { + // Initialize the server by joining itself. glog.V(0).Infoln("Initializing new cluster") _, err := s.raftServer.Do(&raft.DefaultJoinCommand{ @@ -95,7 +99,7 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin } } else { - glog.V(0).Infoln("Recovered from log") + glog.V(0).Infoln("Old conf,log,snapshot should have been removed.") } return s From b0e8f8ca2a63c7be9fe0c32b68e15dc47001a672 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 27 Oct 2014 01:10:33 -0700 Subject: [PATCH 15/67] Upgraded to 0.97 goclipse. --- .project | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.project b/.project index f5551728c1..4dee05c730 100644 --- a/.project +++ b/.project @@ -5,11 +5,6 @@ - - com.googlecode.goclipse.goBuilder - - - goclipse.goNature From cd1b70caf7969a5789f285cdb0c23f9b690626ab Mon Sep 17 00:00:00 2001 From: Yanyi Wu Date: Sat, 1 Nov 2014 01:48:19 +0800 Subject: [PATCH 16/67] Update Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1243727a2e..06d81ebf0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM cydev/go -RUN go get code.google.com/p/weed-fs/go/weed +RUN go get github.com/chrislusf/weed-fs/go/weed EXPOSE 8080 EXPOSE 9333 VOLUME /data From 8af475300295adf9aadc6f57216112db1e6765cd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 28 Nov 2014 16:34:03 -0800 Subject: [PATCH 17/67] Write request id to first 8 bytes of a file, instead of whole file, for better write performance. --- go/util/constants.go | 2 +- go/weed/benchmark.go | 26 ++++++++++++++++---------- go/weed/server.go | 10 +++++----- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/go/util/constants.go b/go/util/constants.go index c407533a85..61f05898ff 100644 --- a/go/util/constants.go +++ b/go/util/constants.go @@ -1,5 +1,5 @@ package util const ( - VERSION = "0.64" + VERSION = "0.66" ) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index 8339913cd4..e5cf831c67 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -31,6 +31,7 @@ type BenchmarkOptions struct { sequentialRead *bool collection *string cpuprofile *string + maxCpu *int vid2server map[string]string //cache for vid locations } @@ -51,7 +52,8 @@ func init() { b.read = cmdBenchmark.Flag.Bool("read", true, "enable read") b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file") b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection") - b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "write cpu profile to file") + b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file") + b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") b.vid2server = make(map[string]string) } @@ -59,25 +61,25 @@ var cmdBenchmark = &Command{ UsageLine: "benchmark -server=localhost:9333 -c=10 -n=100000", Short: "benchmark on writing millions of files and read out", Long: `benchmark on an empty weed file system. - + Two tests during benchmark: 1) write lots of small files to the system 2) read the files out - + The file content is mostly zero, but no compression is done. - + You can choose to only benchmark read or write. During write, the list of uploaded file ids is stored in "-list" specified file. You can also use your own list of file ids to run read test. - + Write speed and read speed will be collected. The numbers are used to get a sense of the system. Usually your network or the hard drive is the real bottleneck. - + Another thing to watch is whether the volumes are evenly distributed to each volume server. Because the 7 more benchmark volumes are randomly distributed to servers with free slots, it's highly possible some servers have uneven amount of - benchmark volumes. To remedy this, you can use this to grow the benchmark volumes + benchmark volumes. To remedy this, you can use this to grow the benchmark volumes before starting the benchmark command: http://localhost:9333/vol/grow?collection=benchmark&count=5 @@ -100,6 +102,10 @@ func init() { func runbenchmark(cmd *Command, args []string) bool { fmt.Printf("This is Seaweed File System version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH) + if *b.maxCpu < 1 { + *b.maxCpu = runtime.NumCPU() + } + runtime.GOMAXPROCS(*b.maxCpu) if *b.cpuprofile != "" { f, err := os.Create(*b.cpuprofile) if err != nil { @@ -497,9 +503,9 @@ func (l *FakeReader) Read(p []byte) (n int, err error) { } else { n = len(p) } - for i := 0; i < n-8; i += 8 { - for s := uint(0); s < 8; s++ { - p[i] = byte(l.id >> (s * 8)) + if n >= 8 { + for i := 0; i < 8; i++ { + p[i] = byte(l.id >> uint(i*8)) } } l.size -= int64(n) diff --git a/go/weed/server.go b/go/weed/server.go index 6042c22e77..38ccdc137b 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -32,17 +32,17 @@ func init() { var cmdServer = &Command{ UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name", Short: "start a server, including volume server, and automatically elect a master server", - Long: `start both a volume server to provide storage spaces + Long: `start both a volume server to provide storage spaces and a master server to provide volume=>location mapping service and sequence number of file ids - + This is provided as a convenient way to start both volume server and master server. The servers are exactly the same as starting them separately. So other volume servers can use this embedded master server also. - + Optionally, one filer server can be started. Logically, filer servers should not be in a cluster. They run with meta data on disk, not shared. So each filer server is different. - + `, } @@ -73,7 +73,7 @@ var ( ) func init() { - serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "write cpu profile to file") + serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file") filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server") filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection") filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") From 9d03cd20ba518e2aad6db6abe8f51ee9445eaacd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 28 Nov 2014 16:55:40 -0800 Subject: [PATCH 18/67] De-support go 1.2 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a6f8b8815d..79cec6d8a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: go go: - - 1.2 - 1.3 - release - tip From ca67ed69a1e80bd1166105fb8a08d8c5b4f9c83f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 28 Nov 2014 17:02:10 -0800 Subject: [PATCH 19/67] Change name to Seaweed. --- docs/benchmarks.rst | 19 +++++++++++++++++-- docs/directories.rst | 2 +- docs/failover.rst | 2 +- docs/gettingstarted.rst | 6 +++--- go/weed/fix.go | 2 +- 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/docs/benchmarks.rst b/docs/benchmarks.rst index e01e6e4bac..d0b71428fe 100644 --- a/docs/benchmarks.rst +++ b/docs/benchmarks.rst @@ -1,7 +1,7 @@ Benchmarks ====================== -Do we really need the benchmark? People always use benchmark to compare systems. But benchmarks are misleading. The resources, e.g., CPU, disk, memory, network, all matter a lot. And with Weed File System, single node vs multiple nodes, benchmarking on one machine vs several multiple machines, all matter a lot. +Do we really need the benchmark? People always use benchmark to compare systems. But benchmarks are misleading. The resources, e.g., CPU, disk, memory, network, all matter a lot. And with Seaweed File System, single node vs multiple nodes, benchmarking on one machine vs several multiple machines, all matter a lot. Here is the steps on how to run benchmark if you really need some numbers. @@ -38,7 +38,22 @@ Many options are options are configurable. Please check the help content: Common Problems ############################### -The most common problem is "too many open files" error. This is because the test itself starts too many network connections on one single machine. In my local macbook, if I ran "random read" following writing right away, the error happens always. I have to run "weed benchmark -write=false" to run the reading test only. Also, changing the concurrency level to "-c=16" would also help. +The most common +I start weed servers in one console for simplicity. Better run servers on different consoles. + +For more realistic tests, please start them on different machines. + +.. code-block:: bash + + # prepare directories + mkdir 3 4 5 + # start 3 servers + ./weed server -dir=./3 -master.port=9333 -volume.port=8083 & + ./weed volume -dir=./4 -port=8084 & + ./weed volume -dir=./5 -port=8085 & + ./weed benchmark -server=localhost:9333 + + problem is "too many open files" error. This is because the test itself starts too many network connections on one single machine. In my local macbook, if I ran "random read" following writing right away, the error happens always. I have to run "weed benchmark -write=false" to run the reading test only. Also, changing the concurrency level to "-c=16" would also help. My own unscientific single machine results ################################################### diff --git a/docs/directories.rst b/docs/directories.rst index 787c7a2abb..aa14da4f72 100644 --- a/docs/directories.rst +++ b/docs/directories.rst @@ -1,7 +1,7 @@ Directories and files =========================== -When talking about file systems, many people would assume directories, list files under a directory, etc. These are expected if we want to hook up Weed File System with linux by FUSE, or with Hadoop, etc. +When talking about file systems, many people would assume directories, list files under a directory, etc. These are expected if we want to hook up Seaweed File System with linux by FUSE, or with Hadoop, etc. Sample usage ##################### diff --git a/docs/failover.rst b/docs/failover.rst index f244918d8e..506a33e3b6 100644 --- a/docs/failover.rst +++ b/docs/failover.rst @@ -7,7 +7,7 @@ Introduction Some user will ask for no single point of failure. Although google runs its file system with a single master for years, no SPOF seems becoming a criteria for architects to pick solutions. -Luckily, it's not too difficult to enable Weed File System with failover master servers. +Luckily, it's not too difficult to enable Seaweed File System with failover master servers. Cheat Sheet: Startup multiple servers ######################################## diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst index 19eb130a72..ad28319fec 100644 --- a/docs/gettingstarted.rst +++ b/docs/gettingstarted.rst @@ -1,6 +1,6 @@ Getting started =================================== -Installing Weed-Fs +Installing Seaweed-FS ################################### Download a proper version from `Seaweed-FS download page `_. @@ -57,7 +57,7 @@ Actually, forget about previous commands. You can setup one master server and on # use "weed server -h" to find out more ./weed server -master.port=9333 -volume.port=8080 -dir="./data" -Testing Weed-Fs +Testing Seaweed-FS ################################### With the master and volume server up, now what? Let's pump in a lot of files into the system! @@ -77,7 +77,7 @@ Then, you can simply check "du -m -s /some/big/folder" to see the actual disk us Now you can use your tools to hit weed-fs as hard as you can. -Using Weed-Fs in docker +Using Seaweed-FS in docker #################################### You can use image "cydev/weed" or build your own with `dockerfile `_ in the root of repo. diff --git a/go/weed/fix.go b/go/weed/fix.go index 9c8026448b..e66075ed25 100644 --- a/go/weed/fix.go +++ b/go/weed/fix.go @@ -17,7 +17,7 @@ func init() { var cmdFix = &Command{ UsageLine: "fix -dir=/tmp -volumeId=234", Short: "run weed tool fix on index file if corrupted", - Long: `Fix runs the WeedFS fix command to re-create the index .idx file. + Long: `Fix runs the SeeweedFS fix command to re-create the index .idx file. `, } From 6c5a3d3dbfc3e801c962bd650cfd4455483cc63c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 30 Nov 2014 21:55:53 -0800 Subject: [PATCH 20/67] Increase performance by reusing []byte, reducing GC. --- go/weed/benchmark.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index e5cf831c67..f88d0d6a9c 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -33,10 +33,12 @@ type BenchmarkOptions struct { cpuprofile *string maxCpu *int vid2server map[string]string //cache for vid locations + } var ( - b BenchmarkOptions + b BenchmarkOptions + sharedBytes []byte ) func init() { @@ -55,6 +57,7 @@ func init() { b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file") b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") b.vid2server = make(map[string]string) + sharedBytes = make([]byte, 1024) } var cmdBenchmark = &Command{ @@ -512,6 +515,23 @@ func (l *FakeReader) Read(p []byte) (n int, err error) { return } +func (l *FakeReader) WriteTo(w io.Writer) (n int64, err error) { + size := int(l.size) + bufferSize := len(sharedBytes) + for size > 0 { + tempBuffer := sharedBytes + if size < bufferSize { + tempBuffer = sharedBytes[0:size] + } + count, e := w.Write(tempBuffer) + if e != nil { + return int64(size), e + } + size -= count + } + return l.size, nil +} + func Readln(r *bufio.Reader) ([]byte, error) { var ( isPrefix bool = true @@ -524,3 +544,4 @@ func Readln(r *bufio.Reader) ([]byte, error) { } return ln, err } + From 74b76a2c41694a74920230afe5fa2559eeb885b3 Mon Sep 17 00:00:00 2001 From: Shaoshan Liu Date: Wed, 3 Dec 2014 21:04:02 -0800 Subject: [PATCH 21/67] add instructions for users who are not familiar with golang --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index ab07683dd1..b22d07569d 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,31 @@ Seaweed File System go get github.com/chrislusf/weed-fs/go/weed ``` +## Installation guide for users who are not familiar with golang + +step 1: install go on your machine and setup the environment by following the instructions from the following link: + +https://golang.org/doc/install + +make sure you set up your $GOPATH + + +step 2: also you may need to install Mercurial by following the instructions below + +http://mercurial.selenic.com/downloads + + +step 3: download, compile, and install the project by executing the following command + +go get github.com/chrislusf/weed-fs/go/weed + +once this is down, you should see the executable "weed" under $GOPATH/bin + +step 4: after you modify your code locally, you could start a local build by calling "go build" under $GOPATH/src/github.com/chrislusf/weed-fs/go/weed + + + + ## Reference For pre-compiled releases, From ed7b00bf0207749cb555d7d77bc35ddf5849ec37 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Wed, 3 Dec 2014 21:37:00 -0800 Subject: [PATCH 22/67] Update README.md --- README.md | 67 ++++++++++++++++++++++--------------------------------- 1 file changed, 27 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index b22d07569d..95d2a5aded 100644 --- a/README.md +++ b/README.md @@ -5,41 +5,6 @@ Seaweed File System [![GoDoc](https://godoc.org/github.com/chrislusf/weed-fs/go?status.svg)](https://godoc.org/github.com/chrislusf/weed-fs/go) [![RTD](https://readthedocs.org/projects/weed-fs/badge/?version=latest)](http://weed-fs.readthedocs.org/en/latest/) -## Usage - -``` -go get github.com/chrislusf/weed-fs/go/weed -``` - -## Installation guide for users who are not familiar with golang - -step 1: install go on your machine and setup the environment by following the instructions from the following link: - -https://golang.org/doc/install - -make sure you set up your $GOPATH - - -step 2: also you may need to install Mercurial by following the instructions below - -http://mercurial.selenic.com/downloads - - -step 3: download, compile, and install the project by executing the following command - -go get github.com/chrislusf/weed-fs/go/weed - -once this is down, you should see the executable "weed" under $GOPATH/bin - -step 4: after you modify your code locally, you could start a local build by calling "go build" under $GOPATH/src/github.com/chrislusf/weed-fs/go/weed - - - - -## Reference - -For pre-compiled releases, - https://bintray.com/chrislusf/Weed-FS/seaweed ## Introduction @@ -271,12 +236,34 @@ More tools and documentation, on how to maintain and scale the system. For examp This is a super exciting project! And I need helpers! -## Contributions ## -To make contributions easier, I have mirrored a repo in github.com -``` - https://github.com/chrislusf/weed-fs.git -``` +## Installation guide for users who are not familiar with golang + +step 1: install go on your machine and setup the environment by following the instructions from the following link: + +https://golang.org/doc/install + +make sure you set up your $GOPATH + + +step 2: also you may need to install Mercurial by following the instructions below + +http://mercurial.selenic.com/downloads + + +step 3: download, compile, and install the project by executing the following command + +go get github.com/chrislusf/weed-fs/go/weed + +once this is done, you should see the executable "weed" under $GOPATH/bin + +step 4: after you modify your code locally, you could start a local build by calling "go install" under $GOPATH/src/github.com/chrislusf/weed-fs/go/weed + +## Reference + +For pre-compiled releases, + https://bintray.com/chrislusf/Weed-FS/seaweed + ## Disk Related topics ## ### Hard Drive Performance ### From 89fd1e4b6e586412f0692ac479b566a323296595 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Thu, 4 Dec 2014 18:30:44 -0800 Subject: [PATCH 23/67] Add more thread safe counters. Tighten thread synchronization. --- go/weed/benchmark.go | 79 +++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index f88d0d6a9c..47b400fcdd 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -132,12 +132,12 @@ func runbenchmark(cmd *Command, args []string) bool { func bench_write() { fileIdLineChan := make(chan string) finishChan := make(chan bool) - writeStats = newStats() + writeStats = newStats(*b.concurrency) idChan := make(chan int) - wait.Add(*b.concurrency) go writeFileIds(*b.idListFile, fileIdLineChan, finishChan) for i := 0; i < *b.concurrency; i++ { - go writeFiles(idChan, fileIdLineChan, writeStats) + wait.Add(1) + go writeFiles(idChan, fileIdLineChan, &writeStats.localStats[i]) } writeStats.start = time.Now() writeStats.total = *b.numberOfFiles @@ -150,7 +150,6 @@ func bench_write() { writeStats.end = time.Now() wait.Add(1) finishChan <- true - finishChan <- true close(finishChan) wait.Wait() writeStats.printStats() @@ -159,14 +158,14 @@ func bench_write() { func bench_read() { fileIdLineChan := make(chan string) finishChan := make(chan bool) - readStats = newStats() - wait.Add(*b.concurrency) + readStats = newStats(*b.concurrency) go readFileIds(*b.idListFile, fileIdLineChan) readStats.start = time.Now() readStats.total = *b.numberOfFiles go readStats.checkProgress("Randomly Reading Benchmark", finishChan) for i := 0; i < *b.concurrency; i++ { - go readFiles(fileIdLineChan, readStats) + wait.Add(1) + go readFiles(fileIdLineChan, &readStats.localStats[i]) } wait.Wait() finishChan <- true @@ -180,12 +179,13 @@ type delayedFile struct { fp *operation.FilePart } -func writeFiles(idChan chan int, fileIdLineChan chan string, s *stats) { +func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { + defer wait.Done() delayedDeleteChan := make(chan *delayedFile, 100) var waitForDeletions sync.WaitGroup for i := 0; i < 7; i++ { + waitForDeletions.Add(1) go func() { - waitForDeletions.Add(1) for df := range delayedDeleteChan { if df == nil { break @@ -228,6 +228,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stats) { s.transferred += fileSize } else { s.failed++ + fmt.Printf("Failed to write with error:%v\n", err) } writeStats.addSample(time.Now().Sub(start)) <-serverLimitChan[fp.Server] @@ -244,10 +245,10 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stats) { } close(delayedDeleteChan) waitForDeletions.Wait() - wait.Done() } -func readFiles(fileIdLineChan chan string, s *stats) { +func readFiles(fileIdLineChan chan string, s *stat) { + defer wait.Done() serverLimitChan := make(map[string]chan bool) masterLimitChan := make(chan bool, 1) for { @@ -288,7 +289,7 @@ func readFiles(fileIdLineChan chan string, s *stats) { readStats.addSample(time.Now().Sub(start)) } else { s.failed++ - println("!!!! Failed to read from ", url, " !!!!!") + fmt.Printf("Failed to read %s error:%v\n", url, err) } <-serverLimitChan[server] } else { @@ -299,7 +300,6 @@ func readFiles(fileIdLineChan chan string, s *stats) { break } } - wait.Done() } func writeFileIds(fileName string, fileIdLineChan chan string, finishChan chan bool) { @@ -363,20 +363,28 @@ const ( // An efficient statics collecting and rendering type stats struct { - data []int - overflow []int + data []int + overflow []int + localStats []stat + start time.Time + end time.Time + total int +} +type stat struct { completed int failed int total int transferred int64 - start time.Time - end time.Time } var percentages = []int{50, 66, 75, 80, 90, 95, 98, 99, 100} -func newStats() *stats { - return &stats{data: make([]int, benchResolution), overflow: make([]int, 0)} +func newStats(n int) *stats { + return &stats{ + data: make([]int, benchResolution), + overflow: make([]int, 0), + localStats: make([]stat, n), + } } func (s *stats) addSample(d time.Duration) { @@ -399,26 +407,38 @@ func (s *stats) checkProgress(testName string, finishChan chan bool) { case <-finishChan: return case t := <-ticker: - completed, transferred, taken := s.completed-lastCompleted, s.transferred-lastTransferred, t.Sub(lastTime) + completed, transferred, taken, total := 0, int64(0), t.Sub(lastTime), s.total + for _, localStat := range s.localStats { + completed += localStat.completed + transferred += localStat.transferred + total += localStat.total + } fmt.Printf("Completed %d of %d requests, %3.1f%% %3.1f/s %3.1fMB/s\n", - s.completed, s.total, float64(s.completed)*100/float64(s.total), - float64(completed)*float64(int64(time.Second))/float64(int64(taken)), - float64(transferred)*float64(int64(time.Second))/float64(int64(taken))/float64(1024*1024), + completed, total, float64(completed)*100/float64(total), + float64(completed-lastCompleted)*float64(int64(time.Second))/float64(int64(taken)), + float64(transferred-lastTransferred)*float64(int64(time.Second))/float64(int64(taken))/float64(1024*1024), ) - lastCompleted, lastTransferred, lastTime = s.completed, s.transferred, t + lastCompleted, lastTransferred, lastTime = completed, transferred, t } } } func (s *stats) printStats() { + completed, failed, transferred, total := 0, 0, int64(0), s.total + for _, localStat := range s.localStats { + completed += localStat.completed + failed += localStat.failed + transferred += localStat.transferred + total += localStat.total + } timeTaken := float64(int64(s.end.Sub(s.start))) / 1000000000 fmt.Printf("\nConcurrency Level: %d\n", *b.concurrency) fmt.Printf("Time taken for tests: %.3f seconds\n", timeTaken) - fmt.Printf("Complete requests: %d\n", s.completed) - fmt.Printf("Failed requests: %d\n", s.failed) - fmt.Printf("Total transferred: %d bytes\n", s.transferred) - fmt.Printf("Requests per second: %.2f [#/sec]\n", float64(s.completed)/timeTaken) - fmt.Printf("Transfer rate: %.2f [Kbytes/sec]\n", float64(s.transferred)/1024/timeTaken) + fmt.Printf("Complete requests: %d\n", completed) + fmt.Printf("Failed requests: %d\n", failed) + fmt.Printf("Total transferred: %d bytes\n", transferred) + fmt.Printf("Requests per second: %.2f [#/sec]\n", float64(completed)/timeTaken) + fmt.Printf("Transfer rate: %.2f [Kbytes/sec]\n", float64(transferred)/1024/timeTaken) n, sum := 0, 0 min, max := 10000000, 0 for i := 0; i < len(s.data); i++ { @@ -544,4 +564,3 @@ func Readln(r *bufio.Reader) ([]byte, error) { } return ln, err } - From 7a6394378cfd3a09bb273211e9bad1ec19e01e84 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Thu, 4 Dec 2014 21:22:09 -0800 Subject: [PATCH 24/67] Remove a volume server concurrent connection limit. --- go/weed/benchmark.go | 150 ++++++++++++++++++------------------------- 1 file changed, 62 insertions(+), 88 deletions(-) diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index 47b400fcdd..f4f0b1874a 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -93,16 +93,11 @@ var cmdBenchmark = &Command{ } var ( - wait sync.WaitGroup - writeStats *stats - readStats *stats - serverLimitChan map[string]chan bool + wait sync.WaitGroup + writeStats *stats + readStats *stats ) -func init() { - serverLimitChan = make(map[string]chan bool) -} - func runbenchmark(cmd *Command, args []string) bool { fmt.Printf("This is Seaweed File System version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH) if *b.maxCpu < 1 { @@ -148,10 +143,11 @@ func bench_write() { close(idChan) wait.Wait() writeStats.end = time.Now() - wait.Add(1) + wait.Add(2) + finishChan <- true finishChan <- true - close(finishChan) wait.Wait() + close(finishChan) writeStats.printStats() } @@ -168,7 +164,9 @@ func bench_read() { go readFiles(fileIdLineChan, &readStats.localStats[i]) } wait.Wait() + wait.Add(1) finishChan <- true + wait.Wait() close(finishChan) readStats.end = time.Now() readStats.printStats() @@ -186,61 +184,46 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { for i := 0; i < 7; i++ { waitForDeletions.Add(1) go func() { + defer waitForDeletions.Done() for df := range delayedDeleteChan { - if df == nil { - break - } if df.enterTime.After(time.Now()) { time.Sleep(df.enterTime.Sub(time.Now())) } - fp := df.fp - serverLimitChan[fp.Server] <- true - if e := util.Delete("http://" + fp.Server + "/" + fp.Fid); e == nil { + if e := util.Delete("http://" + df.fp.Server + "/" + df.fp.Fid); e == nil { s.completed++ } else { s.failed++ } - <-serverLimitChan[fp.Server] } - waitForDeletions.Done() }() } - for { - if id, ok := <-idChan; ok { - start := time.Now() - fileSize := int64(*b.fileSize + rand.Intn(64)) - fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize} - if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection, ""); err == nil { - fp.Server, fp.Fid, fp.Collection = assignResult.PublicUrl, assignResult.Fid, *b.collection - if _, ok := serverLimitChan[fp.Server]; !ok { - serverLimitChan[fp.Server] = make(chan bool, 7) - } - serverLimitChan[fp.Server] <- true - if _, err := fp.Upload(0, *b.server); err == nil { - if rand.Intn(100) < *b.deletePercentage { - s.total++ - delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp} - } else { - fileIdLineChan <- fp.Fid - } - s.completed++ - s.transferred += fileSize + for id := range idChan { + start := time.Now() + fileSize := int64(*b.fileSize + rand.Intn(64)) + fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize} + if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection, ""); err == nil { + fp.Server, fp.Fid, fp.Collection = assignResult.PublicUrl, assignResult.Fid, *b.collection + if _, err := fp.Upload(0, *b.server); err == nil { + if rand.Intn(100) < *b.deletePercentage { + s.total++ + delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp} } else { - s.failed++ - fmt.Printf("Failed to write with error:%v\n", err) - } - writeStats.addSample(time.Now().Sub(start)) - <-serverLimitChan[fp.Server] - if *cmdBenchmark.IsDebug { - fmt.Printf("writing %d file %s\n", id, fp.Fid) + fileIdLineChan <- fp.Fid } + s.completed++ + s.transferred += fileSize } else { s.failed++ - println("writing file error:", err.Error()) + fmt.Printf("Failed to write with error:%v\n", err) + } + writeStats.addSample(time.Now().Sub(start)) + if *cmdBenchmark.IsDebug { + fmt.Printf("writing %d file %s\n", id, fp.Fid) } } else { - break + s.failed++ + println("writing file error:", err.Error()) } } close(delayedDeleteChan) @@ -249,55 +232,45 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { func readFiles(fileIdLineChan chan string, s *stat) { defer wait.Done() - serverLimitChan := make(map[string]chan bool) masterLimitChan := make(chan bool, 1) - for { - if fid, ok := <-fileIdLineChan; ok { - if len(fid) == 0 { - continue - } - if fid[0] == '#' { - continue - } - if *cmdBenchmark.IsDebug { - fmt.Printf("reading file %s\n", fid) - } - parts := strings.SplitN(fid, ",", 2) - vid := parts[0] - start := time.Now() - if server, ok := b.vid2server[vid]; !ok { - masterLimitChan <- true - if _, now_ok := b.vid2server[vid]; !now_ok { - if ret, err := operation.Lookup(*b.server, vid); err == nil { - if len(ret.Locations) > 0 { - server = ret.Locations[0].PublicUrl - b.vid2server[vid] = server - } + for fid := range fileIdLineChan { + if len(fid) == 0 { + continue + } + if fid[0] == '#' { + continue + } + if *cmdBenchmark.IsDebug { + fmt.Printf("reading file %s\n", fid) + } + parts := strings.SplitN(fid, ",", 2) + vid := parts[0] + start := time.Now() + if server, ok := b.vid2server[vid]; !ok { + masterLimitChan <- true + if _, now_ok := b.vid2server[vid]; !now_ok { + if ret, err := operation.Lookup(*b.server, vid); err == nil { + if len(ret.Locations) > 0 { + server = ret.Locations[0].PublicUrl + b.vid2server[vid] = server } } - <-masterLimitChan } - if server, ok := b.vid2server[vid]; ok { - if _, ok := serverLimitChan[server]; !ok { - serverLimitChan[server] = make(chan bool, 7) - } - serverLimitChan[server] <- true - url := "http://" + server + "/" + fid - if bytesRead, err := util.Get(url); err == nil { - s.completed++ - s.transferred += int64(len(bytesRead)) - readStats.addSample(time.Now().Sub(start)) - } else { - s.failed++ - fmt.Printf("Failed to read %s error:%v\n", url, err) - } - <-serverLimitChan[server] + <-masterLimitChan + } + if server, ok := b.vid2server[vid]; ok { + url := "http://" + server + "/" + fid + if bytesRead, err := util.Get(url); err == nil { + s.completed++ + s.transferred += int64(len(bytesRead)) + readStats.addSample(time.Now().Sub(start)) } else { s.failed++ - println("!!!! volume id ", vid, " location not found!!!!!") + fmt.Printf("Failed to read %s error:%v\n", url, err) } } else { - break + s.failed++ + println("!!!! volume id ", vid, " location not found!!!!!") } } } @@ -405,6 +378,7 @@ func (s *stats) checkProgress(testName string, finishChan chan bool) { for { select { case <-finishChan: + wait.Done() return case t := <-ticker: completed, transferred, taken, total := 0, int64(0), t.Sub(lastTime), s.total From 482e3fb97325ae073f19b8293bb25d210beed721 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 5 Dec 2014 12:00:13 -0800 Subject: [PATCH 25/67] Improve Benchmark tool's performance. --- go/util/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/util/constants.go b/go/util/constants.go index 61f05898ff..d677ba44f4 100644 --- a/go/util/constants.go +++ b/go/util/constants.go @@ -1,5 +1,5 @@ package util const ( - VERSION = "0.66" + VERSION = "0.67" ) From ba972694c730429889c696bd9853a38843f64f65 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Mon, 8 Dec 2014 20:27:26 -0800 Subject: [PATCH 26/67] Add filer option to redirect instead of proxying to volume server on file GET requests. --- go/weed/filer.go | 8 +++++-- go/weed/server.go | 6 +++++- go/weed/weed_server/filer_server.go | 22 +++++++++++++------- go/weed/weed_server/filer_server_handlers.go | 12 +++++++++-- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/go/weed/filer.go b/go/weed/filer.go index b3fc9c8788..5ae3a3496c 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -21,6 +21,7 @@ type FilerOptions struct { collection *string defaultReplicaPlacement *string dir *string + redirectOnRead *bool } func init() { @@ -29,7 +30,8 @@ func init() { f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection") f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port") f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data") - f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "Default replication type if not specified.") + f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified") + f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") } var cmdFiler = &Command{ @@ -60,7 +62,9 @@ func runFiler(cmd *Command, args []string) bool { } r := http.NewServeMux() - _, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection) + _, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection, + *f.defaultReplicaPlacement, *f.redirectOnRead, + ) if nfs_err != nil { glog.Fatalf(nfs_err.Error()) } diff --git a/go/weed/server.go b/go/weed/server.go index 38ccdc137b..22222d699c 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -79,6 +79,8 @@ func init() { filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified") filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.") + filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") + } func runServer(cmd *Command, args []string) bool { @@ -150,7 +152,9 @@ func runServer(cmd *Command, args []string) bool { if *isStartingFiler { go func() { r := http.NewServeMux() - _, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection) + _, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection, + *filerOptions.defaultReplicaPlacement, *filerOptions.redirectOnRead, + ) if nfs_err != nil { glog.Fatalf(nfs_err.Error()) } diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index 9d6f7c71be..0bda58d069 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -9,17 +9,23 @@ import ( ) type FilerServer struct { - port string - master string - collection string - filer filer.Filer + port string + master string + collection string + defaultReplication string + redirectOnRead bool + filer filer.Filer } -func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string) (fs *FilerServer, err error) { +func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string, + replication string, redirectOnRead bool, +) (fs *FilerServer, err error) { fs = &FilerServer{ - master: master, - collection: collection, - port: ":" + strconv.Itoa(port), + master: master, + collection: collection, + defaultReplication: replication, + redirectOnRead: redirectOnRead, + port: ":" + strconv.Itoa(port), } if fs.filer, err = filer.NewFilerEmbedded(master, dir); err != nil { diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index 781d7c0fc7..9d8481ac92 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -81,7 +81,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, return } urlLocation := lookup.Locations[rand.Intn(len(lookup.Locations))].PublicUrl - u, _ := url.Parse("http://" + urlLocation + "/" + fileId) + urlString := "http://" + urlLocation + "/" + fileId + if fs.redirectOnRead { + + } + u, _ := url.Parse(urlString) request := &http.Request{ Method: r.Method, URL: u, @@ -110,7 +114,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() - assignResult, ae := operation.Assign(fs.master, 1, query.Get("replication"), fs.collection, query.Get("ttl")) + replication := query.Get("replication") + if replication == "" { + replication = fs.defaultReplication + } + assignResult, ae := operation.Assign(fs.master, 1, replication, fs.collection, query.Get("ttl")) if ae != nil { glog.V(0).Infoln("failing to assign a file id", ae.Error()) writeJsonError(w, r, ae) From 52180f386b044af7a6daba3e33ff04b5e9da25ba Mon Sep 17 00:00:00 2001 From: chrislusf Date: Mon, 8 Dec 2014 20:29:25 -0800 Subject: [PATCH 27/67] Add read-write lock to guard topology changes on new collections and ttls. --- go/topology/collection.go | 21 ++++++++++--------- go/topology/topology.go | 29 +++++++++++++------------- go/topology/topology_map.go | 7 ++++--- go/topology/topology_vacuum.go | 14 +++++++------ go/util/concurrent_read_map.go | 37 ++++++++++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 34 deletions(-) create mode 100644 go/util/concurrent_read_map.go diff --git a/go/topology/collection.go b/go/topology/collection.go index 506f43fbf3..4b47ae88a1 100644 --- a/go/topology/collection.go +++ b/go/topology/collection.go @@ -2,17 +2,18 @@ package topology import ( "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) type Collection struct { Name string volumeSizeLimit uint64 - storageType2VolumeLayout map[string]*VolumeLayout + storageType2VolumeLayout *util.ConcurrentReadMap } func NewCollection(name string, volumeSizeLimit uint64) *Collection { c := &Collection{Name: name, volumeSizeLimit: volumeSizeLimit} - c.storageType2VolumeLayout = make(map[string]*VolumeLayout) + c.storageType2VolumeLayout = util.NewConcurrentReadMap() return c } @@ -21,16 +22,16 @@ func (c *Collection) GetOrCreateVolumeLayout(rp *storage.ReplicaPlacement, ttl * if ttl != nil { keyString += ttl.String() } - if c.storageType2VolumeLayout[keyString] == nil { - c.storageType2VolumeLayout[keyString] = NewVolumeLayout(rp, ttl, c.volumeSizeLimit) - } - return c.storageType2VolumeLayout[keyString] + vl := c.storageType2VolumeLayout.Get(keyString, func() interface{} { + return NewVolumeLayout(rp, ttl, c.volumeSizeLimit) + }) + return vl.(*VolumeLayout) } func (c *Collection) Lookup(vid storage.VolumeId) []*DataNode { - for _, vl := range c.storageType2VolumeLayout { + for _, vl := range c.storageType2VolumeLayout.Items { if vl != nil { - if list := vl.Lookup(vid); list != nil { + if list := vl.(*VolumeLayout).Lookup(vid); list != nil { return list } } @@ -39,9 +40,9 @@ func (c *Collection) Lookup(vid storage.VolumeId) []*DataNode { } func (c *Collection) ListVolumeServers() (nodes []*DataNode) { - for _, vl := range c.storageType2VolumeLayout { + for _, vl := range c.storageType2VolumeLayout.Items { if vl != nil { - if list := vl.ListVolumeServers(); list != nil { + if list := vl.(*VolumeLayout).ListVolumeServers(); list != nil { nodes = append(nodes, list...) } } diff --git a/go/topology/topology.go b/go/topology/topology.go index eb64d336c2..c2073ed2f8 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -9,13 +9,14 @@ import ( "github.com/chrislusf/weed-fs/go/operation" "github.com/chrislusf/weed-fs/go/sequence" "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" "github.com/goraft/raft" ) type Topology struct { NodeImpl - collectionMap map[string]*Collection + collectionMap *util.ConcurrentReadMap pulse int64 @@ -38,7 +39,7 @@ func NewTopology(id string, confFile string, seq sequence.Sequencer, volumeSizeL t.nodeType = "Topology" t.NodeImpl.value = t t.children = make(map[NodeId]Node) - t.collectionMap = make(map[string]*Collection) + t.collectionMap = util.NewConcurrentReadMap() t.pulse = int64(pulse) t.volumeSizeLimit = volumeSizeLimit @@ -90,14 +91,14 @@ func (t *Topology) loadConfiguration(configurationFile string) error { func (t *Topology) Lookup(collection string, vid storage.VolumeId) []*DataNode { //maybe an issue if lots of collections? if collection == "" { - for _, c := range t.collectionMap { - if list := c.Lookup(vid); list != nil { + for _, c := range t.collectionMap.Items { + if list := c.(*Collection).Lookup(vid); list != nil { return list } } } else { - if c, ok := t.collectionMap[collection]; ok { - return c.Lookup(vid) + if c, ok := t.collectionMap.Items[collection]; ok { + return c.(*Collection).Lookup(vid) } } return nil @@ -125,20 +126,18 @@ func (t *Topology) PickForWrite(count int, option *VolumeGrowOption) (string, in } func (t *Topology) GetVolumeLayout(collectionName string, rp *storage.ReplicaPlacement, ttl *storage.TTL) *VolumeLayout { - _, ok := t.collectionMap[collectionName] - if !ok { - t.collectionMap[collectionName] = NewCollection(collectionName, t.volumeSizeLimit) - } - return t.collectionMap[collectionName].GetOrCreateVolumeLayout(rp, ttl) + return t.collectionMap.Get(collectionName, func() interface{} { + return NewCollection(collectionName, t.volumeSizeLimit) + }).(*Collection).GetOrCreateVolumeLayout(rp, ttl) } -func (t *Topology) GetCollection(collectionName string) (collection *Collection, ok bool) { - collection, ok = t.collectionMap[collectionName] - return +func (t *Topology) GetCollection(collectionName string) (*Collection, bool) { + c, hasCollection := t.collectionMap.Items[collectionName] + return c.(*Collection), hasCollection } func (t *Topology) DeleteCollection(collectionName string) { - delete(t.collectionMap, collectionName) + delete(t.collectionMap.Items, collectionName) } func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) { diff --git a/go/topology/topology_map.go b/go/topology/topology_map.go index af95c65367..6a1423ca84 100644 --- a/go/topology/topology_map.go +++ b/go/topology/topology_map.go @@ -11,10 +11,11 @@ func (t *Topology) ToMap() interface{} { } m["DataCenters"] = dcs var layouts []interface{} - for _, c := range t.collectionMap { - for _, layout := range c.storageType2VolumeLayout { + for _, col := range t.collectionMap.Items { + c := col.(*Collection) + for _, layout := range c.storageType2VolumeLayout.Items { if layout != nil { - tmp := layout.ToMap() + tmp := layout.(*VolumeLayout).ToMap() tmp["collection"] = c.Name layouts = append(layouts, tmp) } diff --git a/go/topology/topology_vacuum.go b/go/topology/topology_vacuum.go index 97a76026d7..d6fa2213e7 100644 --- a/go/topology/topology_vacuum.go +++ b/go/topology/topology_vacuum.go @@ -80,13 +80,15 @@ func batchVacuumVolumeCommit(vl *VolumeLayout, vid storage.VolumeId, locationlis return isCommitSuccess } func (t *Topology) Vacuum(garbageThreshold string) int { - for _, c := range t.collectionMap { - for _, vl := range c.storageType2VolumeLayout { + for _, col := range t.collectionMap.Items { + c := col.(*Collection) + for _, vl := range c.storageType2VolumeLayout.Items { if vl != nil { - for vid, locationlist := range vl.vid2location { - if batchVacuumVolumeCheck(vl, vid, locationlist, garbageThreshold) { - if batchVacuumVolumeCompact(vl, vid, locationlist) { - batchVacuumVolumeCommit(vl, vid, locationlist) + volumeLayout := vl.(*VolumeLayout) + for vid, locationlist := range volumeLayout.vid2location { + if batchVacuumVolumeCheck(volumeLayout, vid, locationlist, garbageThreshold) { + if batchVacuumVolumeCompact(volumeLayout, vid, locationlist) { + batchVacuumVolumeCommit(volumeLayout, vid, locationlist) } } } diff --git a/go/util/concurrent_read_map.go b/go/util/concurrent_read_map.go new file mode 100644 index 0000000000..d16fdbcaf3 --- /dev/null +++ b/go/util/concurrent_read_map.go @@ -0,0 +1,37 @@ +package util + +import "sync" + +// A mostly for read map, which can thread-safely +// initialize the map entries. +type ConcurrentReadMap struct { + rmutex sync.RWMutex + mutex sync.Mutex + Items map[string]interface{} +} + +func NewConcurrentReadMap() *ConcurrentReadMap { + return &ConcurrentReadMap{Items: make(map[string]interface{})} +} + +func (m *ConcurrentReadMap) initMapEntry(key string, newEntry func() interface{}) (value interface{}) { + m.mutex.Lock() + defer m.mutex.Unlock() + if value, ok := m.Items[key]; ok { + return value + } + value = newEntry() + m.Items[key] = value + return value +} + +func (m *ConcurrentReadMap) Get(key string, newEntry func() interface{}) interface{} { + m.rmutex.RLock() + if value, ok := m.Items[key]; ok { + m.rmutex.RUnlock() + return value + } else { + m.rmutex.RUnlock() + return m.initMapEntry(key, newEntry) + } +} From e431d4121e8da8d7fc243b29b780c2cd535a4210 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Mon, 8 Dec 2014 20:34:27 -0800 Subject: [PATCH 28/67] Add optional http redirect for filer GET requests. --- go/weed/weed_server/filer_server_handlers.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index 9d8481ac92..6f22912a79 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -83,7 +83,8 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, urlLocation := lookup.Locations[rand.Intn(len(lookup.Locations))].PublicUrl urlString := "http://" + urlLocation + "/" + fileId if fs.redirectOnRead { - + http.Redirect(w, r, urlString, http.StatusFound) + return } u, _ := url.Parse(urlString) request := &http.Request{ From c0dfdf439264964cf702b8a872c59f7a79fbd05b Mon Sep 17 00:00:00 2001 From: Brian McQueen Date: Sun, 14 Dec 2014 00:20:21 -0800 Subject: [PATCH 29/67] switch to my forked repo --- go/filer/client_operations.go | 2 +- go/filer/directory_in_map.go | 2 +- go/filer/filer_embedded.go | 2 +- go/filer/files_in_leveldb.go | 2 +- go/operation/assign_file_id.go | 4 ++-- go/operation/delete_content.go | 2 +- go/operation/list_masters.go | 4 ++-- go/operation/lookup.go | 2 +- go/operation/submit.go | 2 +- go/operation/upload_content.go | 2 +- go/storage/cdb_map.go | 2 +- go/storage/cdb_map_test.go | 2 +- go/storage/compact_map_perf_test.go | 4 ++-- go/storage/compress.go | 2 +- go/storage/crc.go | 2 +- go/storage/file_id.go | 4 ++-- go/storage/needle.go | 6 +++--- go/storage/needle_map.go | 4 ++-- go/storage/needle_read_write.go | 4 ++-- go/storage/store.go | 6 +++--- go/storage/store_vacuum.go | 2 +- go/storage/volume.go | 2 +- go/storage/volume_info.go | 2 +- go/storage/volume_super_block.go | 2 +- go/storage/volume_vacuum.go | 2 +- go/tools/read_index.go | 2 +- go/topology/allocate_volume.go | 4 ++-- go/topology/cluster_commands.go | 4 ++-- go/topology/collection.go | 4 ++-- go/topology/data_node.go | 4 ++-- go/topology/node.go | 4 ++-- go/topology/store_replicate.go | 8 ++++---- go/topology/topology.go | 10 +++++----- go/topology/topology_event_handling.go | 4 ++-- go/topology/topology_vacuum.go | 6 +++--- go/topology/volume_growth.go | 4 ++-- go/topology/volume_growth_test.go | 4 ++-- go/topology/volume_layout.go | 4 ++-- go/util/config.go | 2 +- go/util/file_util.go | 2 +- go/util/net_timeout.go | 2 +- go/weed/benchmark.go | 6 +++--- go/weed/compact.go | 4 ++-- go/weed/download.go | 4 ++-- go/weed/export.go | 4 ++-- go/weed/filer.go | 7 ++++--- go/weed/fix.go | 4 ++-- go/weed/master.go | 6 +++--- go/weed/mount_std.go | 8 ++++---- go/weed/server.go | 6 +++--- go/weed/shell.go | 2 +- go/weed/upload.go | 2 +- go/weed/version.go | 2 +- go/weed/volume.go | 6 +++--- go/weed/volume_test.go | 2 +- go/weed/weed.go | 2 +- go/weed/weed_server/common.go | 10 +++++----- go/weed/weed_server/filer_server.go | 4 ++-- go/weed/weed_server/filer_server_handlers.go | 6 +++--- go/weed/weed_server/filer_server_handlers_admin.go | 2 +- go/weed/weed_server/master_server.go | 8 ++++---- go/weed/weed_server/master_server_handlers.go | 6 +++--- go/weed/weed_server/master_server_handlers_admin.go | 10 +++++----- go/weed/weed_server/raft_server.go | 4 ++-- go/weed/weed_server/raft_server_handlers.go | 4 ++-- go/weed/weed_server/volume_server.go | 4 ++-- go/weed/weed_server/volume_server_handlers.go | 12 ++++++------ go/weed/weed_server/volume_server_handlers_admin.go | 6 +++--- go/weed/weed_server/volume_server_handlers_vacuum.go | 2 +- 69 files changed, 141 insertions(+), 140 deletions(-) diff --git a/go/filer/client_operations.go b/go/filer/client_operations.go index b38368735e..50f95bab1f 100644 --- a/go/filer/client_operations.go +++ b/go/filer/client_operations.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/util" "net/url" ) diff --git a/go/filer/directory_in_map.go b/go/filer/directory_in_map.go index ee601066c9..d3fe7080f9 100644 --- a/go/filer/directory_in_map.go +++ b/go/filer/directory_in_map.go @@ -10,7 +10,7 @@ import ( "strings" "sync" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/util" ) var writeLock sync.Mutex //serialize changes to dir.log diff --git a/go/filer/filer_embedded.go b/go/filer/filer_embedded.go index 0b9b4e6687..58684a4d4e 100644 --- a/go/filer/filer_embedded.go +++ b/go/filer/filer_embedded.go @@ -6,7 +6,7 @@ import ( "path/filepath" "strings" - "github.com/chrislusf/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/operation" ) type FilerEmbedded struct { diff --git a/go/filer/files_in_leveldb.go b/go/filer/files_in_leveldb.go index 781bb0e5f3..ca2c4e796f 100644 --- a/go/filer/files_in_leveldb.go +++ b/go/filer/files_in_leveldb.go @@ -3,7 +3,7 @@ package filer import ( "bytes" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" ) diff --git a/go/operation/assign_file_id.go b/go/operation/assign_file_id.go index 672bfa99c0..4fe571d59b 100644 --- a/go/operation/assign_file_id.go +++ b/go/operation/assign_file_id.go @@ -6,8 +6,8 @@ import ( "net/url" "strconv" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" ) type AssignResult struct { diff --git a/go/operation/delete_content.go b/go/operation/delete_content.go index 416a852b3f..06787dabe5 100644 --- a/go/operation/delete_content.go +++ b/go/operation/delete_content.go @@ -7,7 +7,7 @@ import ( "strings" "sync" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/util" ) type DeleteResult struct { diff --git a/go/operation/list_masters.go b/go/operation/list_masters.go index 7ada942438..542a3cb38a 100644 --- a/go/operation/list_masters.go +++ b/go/operation/list_masters.go @@ -3,8 +3,8 @@ package operation import ( "encoding/json" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" ) type ClusterStatusResult struct { diff --git a/go/operation/lookup.go b/go/operation/lookup.go index e6b6658daa..2fd238e7af 100644 --- a/go/operation/lookup.go +++ b/go/operation/lookup.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/util" ) type Location struct { diff --git a/go/operation/submit.go b/go/operation/submit.go index 62db466172..c4f64f5a24 100644 --- a/go/operation/submit.go +++ b/go/operation/submit.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) type FilePart struct { diff --git a/go/operation/upload_content.go b/go/operation/upload_content.go index 480d76dca6..297d78a047 100644 --- a/go/operation/upload_content.go +++ b/go/operation/upload_content.go @@ -14,7 +14,7 @@ import ( "path/filepath" "strings" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) type UploadResult struct { diff --git a/go/storage/cdb_map.go b/go/storage/cdb_map.go index fbb59e9c08..824672cae1 100644 --- a/go/storage/cdb_map.go +++ b/go/storage/cdb_map.go @@ -7,7 +7,7 @@ import ( "os" "path/filepath" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/util" "github.com/tgulacsi/go-cdb" ) diff --git a/go/storage/cdb_map_test.go b/go/storage/cdb_map_test.go index ed690f44f3..3b13d5b76a 100644 --- a/go/storage/cdb_map_test.go +++ b/go/storage/cdb_map_test.go @@ -6,7 +6,7 @@ import ( "runtime" "testing" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) var testIndexFilename string = "../../test/sample.idx" diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go index f746842259..bebed48baf 100644 --- a/go/storage/compact_map_perf_test.go +++ b/go/storage/compact_map_perf_test.go @@ -5,8 +5,8 @@ import ( "os" "testing" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" ) func TestMemoryUsage(t *testing.T) { diff --git a/go/storage/compress.go b/go/storage/compress.go index 5efc9e1f1f..88a9c9c158 100644 --- a/go/storage/compress.go +++ b/go/storage/compress.go @@ -7,7 +7,7 @@ import ( "io/ioutil" "strings" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) /* diff --git a/go/storage/crc.go b/go/storage/crc.go index af25b9e535..01452dd4ac 100644 --- a/go/storage/crc.go +++ b/go/storage/crc.go @@ -4,7 +4,7 @@ import ( "fmt" "hash/crc32" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/util" ) var table = crc32.MakeTable(crc32.Castagnoli) diff --git a/go/storage/file_id.go b/go/storage/file_id.go index f6e36a98c3..55ccdd110e 100644 --- a/go/storage/file_id.go +++ b/go/storage/file_id.go @@ -5,8 +5,8 @@ import ( "errors" "strings" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" ) type FileId struct { diff --git a/go/storage/needle.go b/go/storage/needle.go index aa3206920e..13ee34bd61 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -11,9 +11,9 @@ import ( "strings" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/images" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/images" + "github.com/mcqueenorama/weed-fs/go/util" ) const ( diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go index 504ca15523..c7150d4d92 100644 --- a/go/storage/needle_map.go +++ b/go/storage/needle_map.go @@ -5,8 +5,8 @@ import ( "io" "os" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" ) type NeedleMapper interface { diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index 663b5abbd1..d4529f92d1 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -6,8 +6,8 @@ import ( "io" "os" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" ) const ( diff --git a/go/storage/store.go b/go/storage/store.go index 80d8a30b81..97dc16f70f 100644 --- a/go/storage/store.go +++ b/go/storage/store.go @@ -10,9 +10,9 @@ import ( "strings" proto "code.google.com/p/goprotobuf/proto" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/util" ) const ( diff --git a/go/storage/store_vacuum.go b/go/storage/store_vacuum.go index 209e3b4b31..0db0830554 100644 --- a/go/storage/store_vacuum.go +++ b/go/storage/store_vacuum.go @@ -4,7 +4,7 @@ import ( "fmt" "strconv" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) { diff --git a/go/storage/volume.go b/go/storage/volume.go index 5b0a836054..540da31409 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -10,7 +10,7 @@ import ( "sync" "time" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) type Volume struct { diff --git a/go/storage/volume_info.go b/go/storage/volume_info.go index 6410c17840..503967c97d 100644 --- a/go/storage/volume_info.go +++ b/go/storage/volume_info.go @@ -1,7 +1,7 @@ package storage import ( - "github.com/chrislusf/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/operation" ) type VolumeInfo struct { diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go index 57e0deea9a..32b799f261 100644 --- a/go/storage/volume_super_block.go +++ b/go/storage/volume_super_block.go @@ -4,7 +4,7 @@ import ( "fmt" "os" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) const ( diff --git a/go/storage/volume_vacuum.go b/go/storage/volume_vacuum.go index 7e026a61d9..28028241a5 100644 --- a/go/storage/volume_vacuum.go +++ b/go/storage/volume_vacuum.go @@ -5,7 +5,7 @@ import ( "os" "time" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) func (v *Volume) garbageLevel() float64 { diff --git a/go/tools/read_index.go b/go/tools/read_index.go index 1104dc3482..ed18167b27 100644 --- a/go/tools/read_index.go +++ b/go/tools/read_index.go @@ -6,7 +6,7 @@ import ( "log" "os" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/storage" ) var ( diff --git a/go/topology/allocate_volume.go b/go/topology/allocate_volume.go index a791b4c1c7..38843dd54e 100644 --- a/go/topology/allocate_volume.go +++ b/go/topology/allocate_volume.go @@ -5,8 +5,8 @@ import ( "errors" "net/url" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/util" ) type AllocateVolumeResult struct { diff --git a/go/topology/cluster_commands.go b/go/topology/cluster_commands.go index cafc52c76a..c88de9e0b8 100644 --- a/go/topology/cluster_commands.go +++ b/go/topology/cluster_commands.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" "github.com/goraft/raft" ) diff --git a/go/topology/collection.go b/go/topology/collection.go index 4b47ae88a1..cded90650a 100644 --- a/go/topology/collection.go +++ b/go/topology/collection.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/util" ) type Collection struct { diff --git a/go/topology/data_node.go b/go/topology/data_node.go index 109bd037fe..c12396e874 100644 --- a/go/topology/data_node.go +++ b/go/topology/data_node.go @@ -3,8 +3,8 @@ package topology import ( "strconv" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) type DataNode struct { diff --git a/go/topology/node.go b/go/topology/node.go index 10955fa728..9c8e7700b8 100644 --- a/go/topology/node.go +++ b/go/topology/node.go @@ -5,8 +5,8 @@ import ( "math/rand" "strings" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) type NodeId string diff --git a/go/topology/store_replicate.go b/go/topology/store_replicate.go index 0c52f9d309..b949b33c38 100644 --- a/go/topology/store_replicate.go +++ b/go/topology/store_replicate.go @@ -5,10 +5,10 @@ import ( "net/http" "strconv" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/util" ) func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.VolumeId, needle *storage.Needle, r *http.Request) (size uint32, errorStatus string) { diff --git a/go/topology/topology.go b/go/topology/topology.go index c2073ed2f8..6c9cbe3da2 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -5,11 +5,11 @@ import ( "io/ioutil" "math/rand" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/sequence" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/sequence" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/util" "github.com/goraft/raft" ) diff --git a/go/topology/topology_event_handling.go b/go/topology/topology_event_handling.go index 7e36568b6a..49e67c8f5e 100644 --- a/go/topology/topology_event_handling.go +++ b/go/topology/topology_event_handling.go @@ -4,8 +4,8 @@ import ( "math/rand" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) func (t *Topology) StartRefreshWritableVolumes(garbageThreshold string) { diff --git a/go/topology/topology_vacuum.go b/go/topology/topology_vacuum.go index d6fa2213e7..9e532876f8 100644 --- a/go/topology/topology_vacuum.go +++ b/go/topology/topology_vacuum.go @@ -6,9 +6,9 @@ import ( "net/url" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/util" ) func batchVacuumVolumeCheck(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList, garbageThreshold string) bool { diff --git a/go/topology/volume_growth.go b/go/topology/volume_growth.go index b1f241990b..02466d5392 100644 --- a/go/topology/volume_growth.go +++ b/go/topology/volume_growth.go @@ -5,8 +5,8 @@ import ( "math/rand" "sync" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) /* diff --git a/go/topology/volume_growth_test.go b/go/topology/volume_growth_test.go index 267b360427..333ab5a3a9 100644 --- a/go/topology/volume_growth_test.go +++ b/go/topology/volume_growth_test.go @@ -5,8 +5,8 @@ import ( "fmt" "testing" - "github.com/chrislusf/weed-fs/go/sequence" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/sequence" + "github.com/mcqueenorama/weed-fs/go/storage" ) var topologyLayout = ` diff --git a/go/topology/volume_layout.go b/go/topology/volume_layout.go index de72bf8952..8c6f80954d 100644 --- a/go/topology/volume_layout.go +++ b/go/topology/volume_layout.go @@ -5,8 +5,8 @@ import ( "math/rand" "sync" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) // mapping from volume to its locations, inverted from server to volume diff --git a/go/util/config.go b/go/util/config.go index 4cf1d7c645..071d731c97 100644 --- a/go/util/config.go +++ b/go/util/config.go @@ -13,7 +13,7 @@ import ( "encoding/json" "os" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) type Config struct { diff --git a/go/util/file_util.go b/go/util/file_util.go index 30e24f001c..176aaac963 100644 --- a/go/util/file_util.go +++ b/go/util/file_util.go @@ -5,7 +5,7 @@ import ( "errors" "os" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) func TestFolderWritable(folder string) (err error) { diff --git a/go/util/net_timeout.go b/go/util/net_timeout.go index f274e48022..4e54b6798f 100644 --- a/go/util/net_timeout.go +++ b/go/util/net_timeout.go @@ -4,7 +4,7 @@ import ( "net" "time" - "github.com/chrislusf/weed-fs/go/stats" + "github.com/mcqueenorama/weed-fs/go/stats" ) // Listener wraps a net.Listener, and gives a place to store the timeout diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index f4f0b1874a..2d507ebb6a 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -14,9 +14,9 @@ import ( "sync" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/util" ) type BenchmarkOptions struct { diff --git a/go/weed/compact.go b/go/weed/compact.go index 71c4ea90f5..1ecb85cf95 100644 --- a/go/weed/compact.go +++ b/go/weed/compact.go @@ -1,8 +1,8 @@ package main import ( - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) func init() { diff --git a/go/weed/download.go b/go/weed/download.go index c782654f5e..82c0fd45ec 100644 --- a/go/weed/download.go +++ b/go/weed/download.go @@ -8,8 +8,8 @@ import ( "path" "strings" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/util" ) var ( diff --git a/go/weed/export.go b/go/weed/export.go index c9cc0e3fe8..472e63aeab 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -11,8 +11,8 @@ import ( "text/template" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) func init() { diff --git a/go/weed/filer.go b/go/weed/filer.go index 5ae3a3496c..173ccddc74 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -6,9 +6,9 @@ import ( "strconv" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/weed/weed_server" ) var ( @@ -57,6 +57,7 @@ var cmdFiler = &Command{ } func runFiler(cmd *Command, args []string) bool { + if err := util.TestFolderWritable(*f.dir); err != nil { glog.Fatalf("Check Meta Folder (-dir) Writable %s : %s", *f.dir, err) } diff --git a/go/weed/fix.go b/go/weed/fix.go index e66075ed25..0f5f0b7f3f 100644 --- a/go/weed/fix.go +++ b/go/weed/fix.go @@ -5,8 +5,8 @@ import ( "path" "strconv" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) func init() { diff --git a/go/weed/master.go b/go/weed/master.go index f88964b6dd..885e6e2079 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -8,9 +8,9 @@ import ( "strings" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/weed/weed_server" "github.com/gorilla/mux" ) diff --git a/go/weed/mount_std.go b/go/weed/mount_std.go index 808c6c563a..e930240b1e 100644 --- a/go/weed/mount_std.go +++ b/go/weed/mount_std.go @@ -9,10 +9,10 @@ import ( "bazil.org/fuse" "bazil.org/fuse/fs" - "github.com/chrislusf/weed-fs/go/filer" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/filer" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/util" ) func runMount(cmd *Command, args []string) bool { diff --git a/go/weed/server.go b/go/weed/server.go index 22222d699c..13e9797bac 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -10,9 +10,9 @@ import ( "sync" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/weed/weed_server" "github.com/gorilla/mux" ) diff --git a/go/weed/shell.go b/go/weed/shell.go index f2c4990eab..04cfb47309 100644 --- a/go/weed/shell.go +++ b/go/weed/shell.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) func init() { diff --git a/go/weed/upload.go b/go/weed/upload.go index 2d67c0bd9b..7623f7d9be 100644 --- a/go/weed/upload.go +++ b/go/weed/upload.go @@ -6,7 +6,7 @@ import ( "os" "path/filepath" - "github.com/chrislusf/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/operation" ) var ( diff --git a/go/weed/version.go b/go/weed/version.go index 8d3a6fed7a..13f7ea23ae 100644 --- a/go/weed/version.go +++ b/go/weed/version.go @@ -4,7 +4,7 @@ import ( "fmt" "runtime" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/util" ) var cmdVersion = &Command{ diff --git a/go/weed/volume.go b/go/weed/volume.go index 212cb4b33a..71a2967025 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -8,9 +8,9 @@ import ( "strings" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/util" - "github.com/chrislusf/weed-fs/go/weed/weed_server" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/weed/weed_server" ) func init() { diff --git a/go/weed/volume_test.go b/go/weed/volume_test.go index ef00a8c7cd..b936ba4f28 100644 --- a/go/weed/volume_test.go +++ b/go/weed/volume_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) func TestXYZ(t *testing.T) { diff --git a/go/weed/weed.go b/go/weed/weed.go index c304b7f35d..0620cdf17d 100644 --- a/go/weed/weed.go +++ b/go/weed/weed.go @@ -13,7 +13,7 @@ import ( "unicode" "unicode/utf8" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) var IsDebug *bool diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index 39b0830e33..55f5e69a4d 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -10,11 +10,11 @@ import ( "strconv" "strings" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/stats" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/util" ) var serverStats *stats.ServerStats diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index 0bda58d069..2630a33983 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -4,8 +4,8 @@ import ( "net/http" "strconv" - "github.com/chrislusf/weed-fs/go/filer" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/filer" + "github.com/mcqueenorama/weed-fs/go/glog" ) type FilerServer struct { diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index 6f22912a79..e5a7a860e7 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -11,9 +11,9 @@ import ( "strconv" "strings" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/util" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/go/weed/weed_server/filer_server_handlers_admin.go b/go/weed/weed_server/filer_server_handlers_admin.go index 5ba12e0b8f..f63701a6b7 100644 --- a/go/weed/weed_server/filer_server_handlers_admin.go +++ b/go/weed/weed_server/filer_server_handlers_admin.go @@ -3,7 +3,7 @@ package weed_server import ( "net/http" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) /* diff --git a/go/weed/weed_server/master_server.go b/go/weed/weed_server/master_server.go index d000cb6107..d96354eb9c 100644 --- a/go/weed/weed_server/master_server.go +++ b/go/weed/weed_server/master_server.go @@ -6,10 +6,10 @@ import ( "net/url" "sync" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/sequence" - "github.com/chrislusf/weed-fs/go/topology" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/sequence" + "github.com/mcqueenorama/weed-fs/go/topology" + "github.com/mcqueenorama/weed-fs/go/util" "github.com/goraft/raft" "github.com/gorilla/mux" ) diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index 7a7a3b70d9..25839669f1 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -5,9 +5,9 @@ import ( "strconv" "strings" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/stats" + "github.com/mcqueenorama/weed-fs/go/storage" ) func (ms *MasterServer) lookupVolumeId(vids []string, collection string) (volumeLocations map[string]operation.LookupResult) { diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index d7124e5676..6ad83784bc 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -9,11 +9,11 @@ import ( "strings" proto "code.google.com/p/goprotobuf/proto" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/topology" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/topology" + "github.com/mcqueenorama/weed-fs/go/util" ) func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go index b9aaef2b05..fb85daecf3 100644 --- a/go/weed/weed_server/raft_server.go +++ b/go/weed/weed_server/raft_server.go @@ -14,8 +14,8 @@ import ( "strings" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/topology" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/topology" "github.com/goraft/raft" "github.com/gorilla/mux" ) diff --git a/go/weed/weed_server/raft_server_handlers.go b/go/weed/weed_server/raft_server_handlers.go index b466d9afac..340a34e7b1 100644 --- a/go/weed/weed_server/raft_server_handlers.go +++ b/go/weed/weed_server/raft_server_handlers.go @@ -6,8 +6,8 @@ import ( "net/http" "strings" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/operation" "github.com/goraft/raft" ) diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go index 0a65fd2f67..3e40a790a0 100644 --- a/go/weed/weed_server/volume_server.go +++ b/go/weed/weed_server/volume_server.go @@ -6,8 +6,8 @@ import ( "strconv" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/storage" ) type VolumeServer struct { diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index 83f6149410..e14bcea9ea 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -9,12 +9,12 @@ import ( "strings" "time" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/images" - "github.com/chrislusf/weed-fs/go/operation" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/storage" - "github.com/chrislusf/weed-fs/go/topology" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/images" + "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/mcqueenorama/weed-fs/go/stats" + "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/mcqueenorama/weed-fs/go/topology" ) var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") diff --git a/go/weed/weed_server/volume_server_handlers_admin.go b/go/weed/weed_server/volume_server_handlers_admin.go index caf4c3be8f..0d3f6ac5ba 100644 --- a/go/weed/weed_server/volume_server_handlers_admin.go +++ b/go/weed/weed_server/volume_server_handlers_admin.go @@ -4,9 +4,9 @@ import ( "net/http" "path/filepath" - "github.com/chrislusf/weed-fs/go/glog" - "github.com/chrislusf/weed-fs/go/stats" - "github.com/chrislusf/weed-fs/go/util" + "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/stats" + "github.com/mcqueenorama/weed-fs/go/util" ) func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/volume_server_handlers_vacuum.go b/go/weed/weed_server/volume_server_handlers_vacuum.go index f115e3b8b1..da7172ae99 100644 --- a/go/weed/weed_server/volume_server_handlers_vacuum.go +++ b/go/weed/weed_server/volume_server_handlers_vacuum.go @@ -3,7 +3,7 @@ package weed_server import ( "net/http" - "github.com/chrislusf/weed-fs/go/glog" + "github.com/mcqueenorama/weed-fs/go/glog" ) func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) { From 57ec736941a554919cfed2bd2ac2bb694095a7d2 Mon Sep 17 00:00:00 2001 From: Brian McQueen Date: Sun, 14 Dec 2014 00:33:16 -0800 Subject: [PATCH 30/67] make it turn on the filer when filer.redirectOnRead is set --- go/weed/server.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go/weed/server.go b/go/weed/server.go index 13e9797bac..6a5a0a9355 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -101,6 +101,10 @@ func runServer(cmd *Command, args []string) bool { } } + if *filerOptions.redirectOnRead { + *isStartingFiler = true + } + *filerOptions.master = *serverPublicIp + ":" + strconv.Itoa(*masterPort) if *filerOptions.defaultReplicaPlacement == "" { From d56c748fa889a119139c3a40528a18e8ce4123ad Mon Sep 17 00:00:00 2001 From: Brian McQueen Date: Sun, 14 Dec 2014 00:35:26 -0800 Subject: [PATCH 31/67] switch it back to chris's repo --- go/filer/client_operations.go | 2 +- go/filer/directory_in_map.go | 2 +- go/filer/filer_embedded.go | 2 +- go/filer/files_in_leveldb.go | 2 +- go/operation/assign_file_id.go | 4 ++-- go/operation/delete_content.go | 2 +- go/operation/list_masters.go | 4 ++-- go/operation/lookup.go | 2 +- go/operation/submit.go | 2 +- go/operation/upload_content.go | 2 +- go/storage/cdb_map.go | 2 +- go/storage/cdb_map_test.go | 2 +- go/storage/compact_map_perf_test.go | 4 ++-- go/storage/compress.go | 2 +- go/storage/crc.go | 2 +- go/storage/file_id.go | 4 ++-- go/storage/needle.go | 6 +++--- go/storage/needle_map.go | 4 ++-- go/storage/needle_read_write.go | 4 ++-- go/storage/store.go | 6 +++--- go/storage/store_vacuum.go | 2 +- go/storage/volume.go | 2 +- go/storage/volume_info.go | 2 +- go/storage/volume_super_block.go | 2 +- go/storage/volume_vacuum.go | 2 +- go/tools/read_index.go | 2 +- go/topology/allocate_volume.go | 4 ++-- go/topology/cluster_commands.go | 4 ++-- go/topology/collection.go | 4 ++-- go/topology/data_node.go | 4 ++-- go/topology/node.go | 4 ++-- go/topology/store_replicate.go | 8 ++++---- go/topology/topology.go | 10 +++++----- go/topology/topology_event_handling.go | 4 ++-- go/topology/topology_vacuum.go | 6 +++--- go/topology/volume_growth.go | 4 ++-- go/topology/volume_growth_test.go | 4 ++-- go/topology/volume_layout.go | 4 ++-- go/util/config.go | 2 +- go/util/file_util.go | 2 +- go/util/net_timeout.go | 2 +- go/weed/benchmark.go | 6 +++--- go/weed/compact.go | 4 ++-- go/weed/download.go | 4 ++-- go/weed/export.go | 4 ++-- go/weed/filer.go | 6 +++--- go/weed/fix.go | 4 ++-- go/weed/master.go | 6 +++--- go/weed/mount_std.go | 8 ++++---- go/weed/server.go | 6 +++--- go/weed/shell.go | 2 +- go/weed/upload.go | 2 +- go/weed/version.go | 2 +- go/weed/volume.go | 6 +++--- go/weed/volume_test.go | 2 +- go/weed/weed.go | 2 +- go/weed/weed_server/common.go | 10 +++++----- go/weed/weed_server/filer_server.go | 4 ++-- go/weed/weed_server/filer_server_handlers.go | 6 +++--- go/weed/weed_server/filer_server_handlers_admin.go | 2 +- go/weed/weed_server/master_server.go | 8 ++++---- go/weed/weed_server/master_server_handlers.go | 6 +++--- go/weed/weed_server/master_server_handlers_admin.go | 10 +++++----- go/weed/weed_server/raft_server.go | 4 ++-- go/weed/weed_server/raft_server_handlers.go | 4 ++-- go/weed/weed_server/volume_server.go | 4 ++-- go/weed/weed_server/volume_server_handlers.go | 12 ++++++------ go/weed/weed_server/volume_server_handlers_admin.go | 6 +++--- go/weed/weed_server/volume_server_handlers_vacuum.go | 2 +- 69 files changed, 140 insertions(+), 140 deletions(-) diff --git a/go/filer/client_operations.go b/go/filer/client_operations.go index 50f95bab1f..b38368735e 100644 --- a/go/filer/client_operations.go +++ b/go/filer/client_operations.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" "net/url" ) diff --git a/go/filer/directory_in_map.go b/go/filer/directory_in_map.go index d3fe7080f9..ee601066c9 100644 --- a/go/filer/directory_in_map.go +++ b/go/filer/directory_in_map.go @@ -10,7 +10,7 @@ import ( "strings" "sync" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" ) var writeLock sync.Mutex //serialize changes to dir.log diff --git a/go/filer/filer_embedded.go b/go/filer/filer_embedded.go index 58684a4d4e..0b9b4e6687 100644 --- a/go/filer/filer_embedded.go +++ b/go/filer/filer_embedded.go @@ -6,7 +6,7 @@ import ( "path/filepath" "strings" - "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/operation" ) type FilerEmbedded struct { diff --git a/go/filer/files_in_leveldb.go b/go/filer/files_in_leveldb.go index ca2c4e796f..781bb0e5f3 100644 --- a/go/filer/files_in_leveldb.go +++ b/go/filer/files_in_leveldb.go @@ -3,7 +3,7 @@ package filer import ( "bytes" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" ) diff --git a/go/operation/assign_file_id.go b/go/operation/assign_file_id.go index 4fe571d59b..672bfa99c0 100644 --- a/go/operation/assign_file_id.go +++ b/go/operation/assign_file_id.go @@ -6,8 +6,8 @@ import ( "net/url" "strconv" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type AssignResult struct { diff --git a/go/operation/delete_content.go b/go/operation/delete_content.go index 06787dabe5..416a852b3f 100644 --- a/go/operation/delete_content.go +++ b/go/operation/delete_content.go @@ -7,7 +7,7 @@ import ( "strings" "sync" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" ) type DeleteResult struct { diff --git a/go/operation/list_masters.go b/go/operation/list_masters.go index 542a3cb38a..7ada942438 100644 --- a/go/operation/list_masters.go +++ b/go/operation/list_masters.go @@ -3,8 +3,8 @@ package operation import ( "encoding/json" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type ClusterStatusResult struct { diff --git a/go/operation/lookup.go b/go/operation/lookup.go index 2fd238e7af..e6b6658daa 100644 --- a/go/operation/lookup.go +++ b/go/operation/lookup.go @@ -9,7 +9,7 @@ import ( "strings" "time" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" ) type Location struct { diff --git a/go/operation/submit.go b/go/operation/submit.go index c4f64f5a24..62db466172 100644 --- a/go/operation/submit.go +++ b/go/operation/submit.go @@ -9,7 +9,7 @@ import ( "strconv" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) type FilePart struct { diff --git a/go/operation/upload_content.go b/go/operation/upload_content.go index 297d78a047..480d76dca6 100644 --- a/go/operation/upload_content.go +++ b/go/operation/upload_content.go @@ -14,7 +14,7 @@ import ( "path/filepath" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) type UploadResult struct { diff --git a/go/storage/cdb_map.go b/go/storage/cdb_map.go index 824672cae1..fbb59e9c08 100644 --- a/go/storage/cdb_map.go +++ b/go/storage/cdb_map.go @@ -7,7 +7,7 @@ import ( "os" "path/filepath" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" "github.com/tgulacsi/go-cdb" ) diff --git a/go/storage/cdb_map_test.go b/go/storage/cdb_map_test.go index 3b13d5b76a..ed690f44f3 100644 --- a/go/storage/cdb_map_test.go +++ b/go/storage/cdb_map_test.go @@ -6,7 +6,7 @@ import ( "runtime" "testing" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) var testIndexFilename string = "../../test/sample.idx" diff --git a/go/storage/compact_map_perf_test.go b/go/storage/compact_map_perf_test.go index bebed48baf..f746842259 100644 --- a/go/storage/compact_map_perf_test.go +++ b/go/storage/compact_map_perf_test.go @@ -5,8 +5,8 @@ import ( "os" "testing" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) func TestMemoryUsage(t *testing.T) { diff --git a/go/storage/compress.go b/go/storage/compress.go index 88a9c9c158..5efc9e1f1f 100644 --- a/go/storage/compress.go +++ b/go/storage/compress.go @@ -7,7 +7,7 @@ import ( "io/ioutil" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) /* diff --git a/go/storage/crc.go b/go/storage/crc.go index 01452dd4ac..af25b9e535 100644 --- a/go/storage/crc.go +++ b/go/storage/crc.go @@ -4,7 +4,7 @@ import ( "fmt" "hash/crc32" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" ) var table = crc32.MakeTable(crc32.Castagnoli) diff --git a/go/storage/file_id.go b/go/storage/file_id.go index 55ccdd110e..f6e36a98c3 100644 --- a/go/storage/file_id.go +++ b/go/storage/file_id.go @@ -5,8 +5,8 @@ import ( "errors" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type FileId struct { diff --git a/go/storage/needle.go b/go/storage/needle.go index 13ee34bd61..aa3206920e 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -11,9 +11,9 @@ import ( "strings" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/images" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/images" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/needle_map.go b/go/storage/needle_map.go index c7150d4d92..504ca15523 100644 --- a/go/storage/needle_map.go +++ b/go/storage/needle_map.go @@ -5,8 +5,8 @@ import ( "io" "os" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) type NeedleMapper interface { diff --git a/go/storage/needle_read_write.go b/go/storage/needle_read_write.go index d4529f92d1..663b5abbd1 100644 --- a/go/storage/needle_read_write.go +++ b/go/storage/needle_read_write.go @@ -6,8 +6,8 @@ import ( "io" "os" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/store.go b/go/storage/store.go index 97dc16f70f..80d8a30b81 100644 --- a/go/storage/store.go +++ b/go/storage/store.go @@ -10,9 +10,9 @@ import ( "strings" proto "code.google.com/p/goprotobuf/proto" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) const ( diff --git a/go/storage/store_vacuum.go b/go/storage/store_vacuum.go index 0db0830554..209e3b4b31 100644 --- a/go/storage/store_vacuum.go +++ b/go/storage/store_vacuum.go @@ -4,7 +4,7 @@ import ( "fmt" "strconv" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) func (s *Store) CheckCompactVolume(volumeIdString string, garbageThresholdString string) (error, bool) { diff --git a/go/storage/volume.go b/go/storage/volume.go index 540da31409..5b0a836054 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -10,7 +10,7 @@ import ( "sync" "time" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) type Volume struct { diff --git a/go/storage/volume_info.go b/go/storage/volume_info.go index 503967c97d..6410c17840 100644 --- a/go/storage/volume_info.go +++ b/go/storage/volume_info.go @@ -1,7 +1,7 @@ package storage import ( - "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/operation" ) type VolumeInfo struct { diff --git a/go/storage/volume_super_block.go b/go/storage/volume_super_block.go index 32b799f261..57e0deea9a 100644 --- a/go/storage/volume_super_block.go +++ b/go/storage/volume_super_block.go @@ -4,7 +4,7 @@ import ( "fmt" "os" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) const ( diff --git a/go/storage/volume_vacuum.go b/go/storage/volume_vacuum.go index 28028241a5..7e026a61d9 100644 --- a/go/storage/volume_vacuum.go +++ b/go/storage/volume_vacuum.go @@ -5,7 +5,7 @@ import ( "os" "time" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) func (v *Volume) garbageLevel() float64 { diff --git a/go/tools/read_index.go b/go/tools/read_index.go index ed18167b27..1104dc3482 100644 --- a/go/tools/read_index.go +++ b/go/tools/read_index.go @@ -6,7 +6,7 @@ import ( "log" "os" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/storage" ) var ( diff --git a/go/topology/allocate_volume.go b/go/topology/allocate_volume.go index 38843dd54e..a791b4c1c7 100644 --- a/go/topology/allocate_volume.go +++ b/go/topology/allocate_volume.go @@ -5,8 +5,8 @@ import ( "errors" "net/url" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) type AllocateVolumeResult struct { diff --git a/go/topology/cluster_commands.go b/go/topology/cluster_commands.go index c88de9e0b8..cafc52c76a 100644 --- a/go/topology/cluster_commands.go +++ b/go/topology/cluster_commands.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" "github.com/goraft/raft" ) diff --git a/go/topology/collection.go b/go/topology/collection.go index cded90650a..4b47ae88a1 100644 --- a/go/topology/collection.go +++ b/go/topology/collection.go @@ -1,8 +1,8 @@ package topology import ( - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) type Collection struct { diff --git a/go/topology/data_node.go b/go/topology/data_node.go index c12396e874..109bd037fe 100644 --- a/go/topology/data_node.go +++ b/go/topology/data_node.go @@ -3,8 +3,8 @@ package topology import ( "strconv" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) type DataNode struct { diff --git a/go/topology/node.go b/go/topology/node.go index 9c8e7700b8..10955fa728 100644 --- a/go/topology/node.go +++ b/go/topology/node.go @@ -5,8 +5,8 @@ import ( "math/rand" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) type NodeId string diff --git a/go/topology/store_replicate.go b/go/topology/store_replicate.go index b949b33c38..0c52f9d309 100644 --- a/go/topology/store_replicate.go +++ b/go/topology/store_replicate.go @@ -5,10 +5,10 @@ import ( "net/http" "strconv" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) func ReplicatedWrite(masterNode string, s *storage.Store, volumeId storage.VolumeId, needle *storage.Needle, r *http.Request) (size uint32, errorStatus string) { diff --git a/go/topology/topology.go b/go/topology/topology.go index 6c9cbe3da2..c2073ed2f8 100644 --- a/go/topology/topology.go +++ b/go/topology/topology.go @@ -5,11 +5,11 @@ import ( "io/ioutil" "math/rand" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/sequence" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/sequence" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" "github.com/goraft/raft" ) diff --git a/go/topology/topology_event_handling.go b/go/topology/topology_event_handling.go index 49e67c8f5e..7e36568b6a 100644 --- a/go/topology/topology_event_handling.go +++ b/go/topology/topology_event_handling.go @@ -4,8 +4,8 @@ import ( "math/rand" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func (t *Topology) StartRefreshWritableVolumes(garbageThreshold string) { diff --git a/go/topology/topology_vacuum.go b/go/topology/topology_vacuum.go index 9e532876f8..d6fa2213e7 100644 --- a/go/topology/topology_vacuum.go +++ b/go/topology/topology_vacuum.go @@ -6,9 +6,9 @@ import ( "net/url" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) func batchVacuumVolumeCheck(vl *VolumeLayout, vid storage.VolumeId, locationlist *VolumeLocationList, garbageThreshold string) bool { diff --git a/go/topology/volume_growth.go b/go/topology/volume_growth.go index 02466d5392..b1f241990b 100644 --- a/go/topology/volume_growth.go +++ b/go/topology/volume_growth.go @@ -5,8 +5,8 @@ import ( "math/rand" "sync" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) /* diff --git a/go/topology/volume_growth_test.go b/go/topology/volume_growth_test.go index 333ab5a3a9..267b360427 100644 --- a/go/topology/volume_growth_test.go +++ b/go/topology/volume_growth_test.go @@ -5,8 +5,8 @@ import ( "fmt" "testing" - "github.com/mcqueenorama/weed-fs/go/sequence" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/sequence" + "github.com/chrislusf/weed-fs/go/storage" ) var topologyLayout = ` diff --git a/go/topology/volume_layout.go b/go/topology/volume_layout.go index 8c6f80954d..de72bf8952 100644 --- a/go/topology/volume_layout.go +++ b/go/topology/volume_layout.go @@ -5,8 +5,8 @@ import ( "math/rand" "sync" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) // mapping from volume to its locations, inverted from server to volume diff --git a/go/util/config.go b/go/util/config.go index 071d731c97..4cf1d7c645 100644 --- a/go/util/config.go +++ b/go/util/config.go @@ -13,7 +13,7 @@ import ( "encoding/json" "os" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) type Config struct { diff --git a/go/util/file_util.go b/go/util/file_util.go index 176aaac963..30e24f001c 100644 --- a/go/util/file_util.go +++ b/go/util/file_util.go @@ -5,7 +5,7 @@ import ( "errors" "os" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) func TestFolderWritable(folder string) (err error) { diff --git a/go/util/net_timeout.go b/go/util/net_timeout.go index 4e54b6798f..f274e48022 100644 --- a/go/util/net_timeout.go +++ b/go/util/net_timeout.go @@ -4,7 +4,7 @@ import ( "net" "time" - "github.com/mcqueenorama/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/stats" ) // Listener wraps a net.Listener, and gives a place to store the timeout diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index 2d507ebb6a..f4f0b1874a 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -14,9 +14,9 @@ import ( "sync" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) type BenchmarkOptions struct { diff --git a/go/weed/compact.go b/go/weed/compact.go index 1ecb85cf95..71c4ea90f5 100644 --- a/go/weed/compact.go +++ b/go/weed/compact.go @@ -1,8 +1,8 @@ package main import ( - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func init() { diff --git a/go/weed/download.go b/go/weed/download.go index 82c0fd45ec..c782654f5e 100644 --- a/go/weed/download.go +++ b/go/weed/download.go @@ -8,8 +8,8 @@ import ( "path" "strings" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" ) var ( diff --git a/go/weed/export.go b/go/weed/export.go index 472e63aeab..c9cc0e3fe8 100644 --- a/go/weed/export.go +++ b/go/weed/export.go @@ -11,8 +11,8 @@ import ( "text/template" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func init() { diff --git a/go/weed/filer.go b/go/weed/filer.go index 173ccddc74..11154f1835 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -6,9 +6,9 @@ import ( "strconv" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" - "github.com/mcqueenorama/weed-fs/go/weed/weed_server" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" ) var ( diff --git a/go/weed/fix.go b/go/weed/fix.go index 0f5f0b7f3f..e66075ed25 100644 --- a/go/weed/fix.go +++ b/go/weed/fix.go @@ -5,8 +5,8 @@ import ( "path" "strconv" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) func init() { diff --git a/go/weed/master.go b/go/weed/master.go index 885e6e2079..f88964b6dd 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -8,9 +8,9 @@ import ( "strings" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" - "github.com/mcqueenorama/weed-fs/go/weed/weed_server" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" "github.com/gorilla/mux" ) diff --git a/go/weed/mount_std.go b/go/weed/mount_std.go index e930240b1e..808c6c563a 100644 --- a/go/weed/mount_std.go +++ b/go/weed/mount_std.go @@ -9,10 +9,10 @@ import ( "bazil.org/fuse" "bazil.org/fuse/fs" - "github.com/mcqueenorama/weed-fs/go/filer" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/filer" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) func runMount(cmd *Command, args []string) bool { diff --git a/go/weed/server.go b/go/weed/server.go index 6a5a0a9355..67bbdb3708 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -10,9 +10,9 @@ import ( "sync" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" - "github.com/mcqueenorama/weed-fs/go/weed/weed_server" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" "github.com/gorilla/mux" ) diff --git a/go/weed/shell.go b/go/weed/shell.go index 04cfb47309..f2c4990eab 100644 --- a/go/weed/shell.go +++ b/go/weed/shell.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) func init() { diff --git a/go/weed/upload.go b/go/weed/upload.go index 7623f7d9be..2d67c0bd9b 100644 --- a/go/weed/upload.go +++ b/go/weed/upload.go @@ -6,7 +6,7 @@ import ( "os" "path/filepath" - "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/operation" ) var ( diff --git a/go/weed/version.go b/go/weed/version.go index 13f7ea23ae..8d3a6fed7a 100644 --- a/go/weed/version.go +++ b/go/weed/version.go @@ -4,7 +4,7 @@ import ( "fmt" "runtime" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/util" ) var cmdVersion = &Command{ diff --git a/go/weed/volume.go b/go/weed/volume.go index 71a2967025..212cb4b33a 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -8,9 +8,9 @@ import ( "strings" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/util" - "github.com/mcqueenorama/weed-fs/go/weed/weed_server" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/weed/weed_server" ) func init() { diff --git a/go/weed/volume_test.go b/go/weed/volume_test.go index b936ba4f28..ef00a8c7cd 100644 --- a/go/weed/volume_test.go +++ b/go/weed/volume_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) func TestXYZ(t *testing.T) { diff --git a/go/weed/weed.go b/go/weed/weed.go index 0620cdf17d..c304b7f35d 100644 --- a/go/weed/weed.go +++ b/go/weed/weed.go @@ -13,7 +13,7 @@ import ( "unicode" "unicode/utf8" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) var IsDebug *bool diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index 55f5e69a4d..39b0830e33 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -10,11 +10,11 @@ import ( "strconv" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/stats" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/util" ) var serverStats *stats.ServerStats diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index 2630a33983..0bda58d069 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -4,8 +4,8 @@ import ( "net/http" "strconv" - "github.com/mcqueenorama/weed-fs/go/filer" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/filer" + "github.com/chrislusf/weed-fs/go/glog" ) type FilerServer struct { diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index e5a7a860e7..6f22912a79 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -11,9 +11,9 @@ import ( "strconv" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/util" "github.com/syndtr/goleveldb/leveldb" ) diff --git a/go/weed/weed_server/filer_server_handlers_admin.go b/go/weed/weed_server/filer_server_handlers_admin.go index f63701a6b7..5ba12e0b8f 100644 --- a/go/weed/weed_server/filer_server_handlers_admin.go +++ b/go/weed/weed_server/filer_server_handlers_admin.go @@ -3,7 +3,7 @@ package weed_server import ( "net/http" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) /* diff --git a/go/weed/weed_server/master_server.go b/go/weed/weed_server/master_server.go index d96354eb9c..d000cb6107 100644 --- a/go/weed/weed_server/master_server.go +++ b/go/weed/weed_server/master_server.go @@ -6,10 +6,10 @@ import ( "net/url" "sync" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/sequence" - "github.com/mcqueenorama/weed-fs/go/topology" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/sequence" + "github.com/chrislusf/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/util" "github.com/goraft/raft" "github.com/gorilla/mux" ) diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index 25839669f1..7a7a3b70d9 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -5,9 +5,9 @@ import ( "strconv" "strings" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/stats" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" ) func (ms *MasterServer) lookupVolumeId(vids []string, collection string) (volumeLocations map[string]operation.LookupResult) { diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index 6ad83784bc..d7124e5676 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -9,11 +9,11 @@ import ( "strings" proto "code.google.com/p/goprotobuf/proto" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/topology" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/util" ) func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go index fb85daecf3..b9aaef2b05 100644 --- a/go/weed/weed_server/raft_server.go +++ b/go/weed/weed_server/raft_server.go @@ -14,8 +14,8 @@ import ( "strings" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/topology" "github.com/goraft/raft" "github.com/gorilla/mux" ) diff --git a/go/weed/weed_server/raft_server_handlers.go b/go/weed/weed_server/raft_server_handlers.go index 340a34e7b1..b466d9afac 100644 --- a/go/weed/weed_server/raft_server_handlers.go +++ b/go/weed/weed_server/raft_server_handlers.go @@ -6,8 +6,8 @@ import ( "net/http" "strings" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/operation" "github.com/goraft/raft" ) diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go index 3e40a790a0..0a65fd2f67 100644 --- a/go/weed/weed_server/volume_server.go +++ b/go/weed/weed_server/volume_server.go @@ -6,8 +6,8 @@ import ( "strconv" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/storage" ) type VolumeServer struct { diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index e14bcea9ea..83f6149410 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -9,12 +9,12 @@ import ( "strings" "time" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/images" - "github.com/mcqueenorama/weed-fs/go/operation" - "github.com/mcqueenorama/weed-fs/go/stats" - "github.com/mcqueenorama/weed-fs/go/storage" - "github.com/mcqueenorama/weed-fs/go/topology" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/images" + "github.com/chrislusf/weed-fs/go/operation" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/storage" + "github.com/chrislusf/weed-fs/go/topology" ) var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") diff --git a/go/weed/weed_server/volume_server_handlers_admin.go b/go/weed/weed_server/volume_server_handlers_admin.go index 0d3f6ac5ba..caf4c3be8f 100644 --- a/go/weed/weed_server/volume_server_handlers_admin.go +++ b/go/weed/weed_server/volume_server_handlers_admin.go @@ -4,9 +4,9 @@ import ( "net/http" "path/filepath" - "github.com/mcqueenorama/weed-fs/go/glog" - "github.com/mcqueenorama/weed-fs/go/stats" - "github.com/mcqueenorama/weed-fs/go/util" + "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/stats" + "github.com/chrislusf/weed-fs/go/util" ) func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { diff --git a/go/weed/weed_server/volume_server_handlers_vacuum.go b/go/weed/weed_server/volume_server_handlers_vacuum.go index da7172ae99..f115e3b8b1 100644 --- a/go/weed/weed_server/volume_server_handlers_vacuum.go +++ b/go/weed/weed_server/volume_server_handlers_vacuum.go @@ -3,7 +3,7 @@ package weed_server import ( "net/http" - "github.com/mcqueenorama/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/glog" ) func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) { From 6ec6541a0db41fae535c895178ce7ebadd94b513 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Sun, 21 Dec 2014 17:49:38 -0800 Subject: [PATCH 32/67] Update index.rst --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 800e9c3f4e..06fdeca7e8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,7 +15,7 @@ For documents and bug reporting, Please visit http://weed-fs.googlecode.com For pre-compiled releases, - https://bintray.com/chrislusf/Weed-FS/weed + https://bintray.com/chrislusf/Weed-FS/seaweed Contents: From 01d96433b1115793e48bb703deb67e3f7f310d5c Mon Sep 17 00:00:00 2001 From: chrislusf Date: Sun, 21 Dec 2014 18:01:31 -0800 Subject: [PATCH 33/67] add ttl.rst --- docs/ttl.rst | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 docs/ttl.rst diff --git a/docs/ttl.rst b/docs/ttl.rst new file mode 100644 index 0000000000..8312c518ad --- /dev/null +++ b/docs/ttl.rst @@ -0,0 +1,81 @@ +Store file with a Time To Live (TTL). +=================== + +Introduction +############################# + +Seaweed is a key~file store, and files can optionally expire with a Time To Live (TTL). + +How to use it? +############################# + +Assume we want to store a file with TTL of 3 minutes. + +First, ask the master to assign a file id to a volume with a 3-minute TTL: +.. code-block:: bash + > curl http://localhost:9333/dir/assign?ttl=3m + {"count":1,"fid":"5,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"} + +Secondly, use the file id to store on the volume server +.. code-block:: bash + > curl -F "file=@x.go" http://127.0.0.1:8080/5,01637037d6?ttl=3m + +After writing, the file content will be returned as usual if read before the TTL expiry. But if read after the TTL expiry, the file will be reported as missing and return the http response status as not found. + +For next writes with ttl=3m, the same set of volumes with ttl=3m will be used until: + 1. the ttl=3m volumes are full. If so, new volumes will be created. + 1. there are no write activities for 3 minutes. If so, these volumes will be stopped and deleted. + +Advanced Usage +############################# + +As you may have noticed, the "ttl=3m" is used twice! One for assigning file id, and one for uploading the actual file. The first one is for master to pick a matching volume, while the second one is written together with the file. + +These two TTL values are not required to be the same. As long as the volume TTL is larger than file TTL, it should be OK. + +This gives some flexibility to fine-tune the file TTL, while reducing the number of volume TTL variations, which simplifies managing the TTL volumes. + +Supported TTL format +############################# + +The TTL is in the format of one integer number followed by one unit. The unit can be 'm', 'h', 'd', 'w', 'M', 'y'. + +Supported TTL format examples: +.. code-block:: bash + 3m: 3 minutes + 4h: 4 hours + 5d: 5 days + 6w: 6 weeks + 7M: 7 months + 8y: 8 years + + +How efficient it is? +############################# + +TTL seems easy to implement since we just need to report the file as missing if the time is over the TTL. However, the real difficulty is to efficiently reclaim disk space from expired files, similar to JVM memory garbage collection, which is a sophisticated piece of work with many man-years of work. + +Memcached also supports TTL. It gets around this problem by putting entries into fix-sized slabs. If one slab is expired, no work is required and the slab can be overwritten right away. However, this fix-sized slab approach wastes disk spaces since the file contents rarely fit in slabs exactly. And the slab managements are not trivial either. + +Seaweed-FS efficiently resolves this disk space garbage collection problem with great simplicity. One of key differences from "normal" implementation is that the TTL is associated with the volume, together with each specific file. + +During the file id assigning step, the file id will be assigned to a volume with matching TTL. The volumes are checked periodically (every 5~10 seconds by default). If the latest expiration time has been reached, all the files in the whole volume will be all expired, and the volume can be safely deleted. + +Implementation Details +############################# + + 1. When assigning file key, the master would pick one TTL volume with matching TTL. If such volumes do not exist, create a few. + 1. Volume servers will write the file with expiration time. When serving file, if the file is expired, the file will be reported as not found. + 1. Volume servers will track each volume's largest expiration time, and stop reporting the expired volumes to the master server. + 1. Master server will think the previously existed volumes are dead, and stop assigning write requests to them. + 1. After about 10% of the TTL time, or at most 10 minutes, the volume servers will delete the expired volume. + +Deployment +############################# + +For deploying to production, the TTL volume maximum size should be taken into consideration. If the writes are frequent, the TTL volume will grow to the max volume size. So when the disk space is not ample enough, it's better to reduce the maximum volume size. + +It's recommended not to mix the TTL volumes and non TTL volumes in the same cluster. This is because the volume maximum size, default to 30GB, is configured on the volume master at the cluster level. + +We could implement the configuration for max volume size for each TTL. However, it could get fairly verbose. Maybe later if it is strongly desired. + From 305c36ddd1b3fc021e0cf4ae77f0e8e65079d1d0 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Sun, 21 Dec 2014 18:08:07 -0800 Subject: [PATCH 34/67] Update ttl.rst --- docs/ttl.rst | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/ttl.rst b/docs/ttl.rst index 8312c518ad..0d36ec547c 100644 --- a/docs/ttl.rst +++ b/docs/ttl.rst @@ -23,8 +23,9 @@ Secondly, use the file id to store on the volume server After writing, the file content will be returned as usual if read before the TTL expiry. But if read after the TTL expiry, the file will be reported as missing and return the http response status as not found. For next writes with ttl=3m, the same set of volumes with ttl=3m will be used until: - 1. the ttl=3m volumes are full. If so, new volumes will be created. - 1. there are no write activities for 3 minutes. If so, these volumes will be stopped and deleted. + +1. the ttl=3m volumes are full. If so, new volumes will be created. +2. there are no write activities for 3 minutes. If so, these volumes will be stopped and deleted. Advanced Usage ############################# @@ -41,13 +42,13 @@ Supported TTL format The TTL is in the format of one integer number followed by one unit. The unit can be 'm', 'h', 'd', 'w', 'M', 'y'. Supported TTL format examples: -.. code-block:: bash - 3m: 3 minutes - 4h: 4 hours - 5d: 5 days - 6w: 6 weeks - 7M: 7 months - 8y: 8 years + +- 3m: 3 minutes +- 4h: 4 hours +- 5d: 5 days +- 6w: 6 weeks +- 7M: 7 months +- 8y: 8 years How efficient it is? @@ -63,12 +64,11 @@ During the file id assigning step, the file id will be assigned to a volume with Implementation Details ############################# - - 1. When assigning file key, the master would pick one TTL volume with matching TTL. If such volumes do not exist, create a few. - 1. Volume servers will write the file with expiration time. When serving file, if the file is expired, the file will be reported as not found. - 1. Volume servers will track each volume's largest expiration time, and stop reporting the expired volumes to the master server. - 1. Master server will think the previously existed volumes are dead, and stop assigning write requests to them. - 1. After about 10% of the TTL time, or at most 10 minutes, the volume servers will delete the expired volume. +1. When assigning file key, the master would pick one TTL volume with matching TTL. If such volumes do not exist, create a few. +2. Volume servers will write the file with expiration time. When serving file, if the file is expired, the file will be reported as not found. +3. Volume servers will track each volume's largest expiration time, and stop reporting the expired volumes to the master server. +4. Master server will think the previously existed volumes are dead, and stop assigning write requests to them. +5. After about 10% of the TTL time, or at most 10 minutes, the volume servers will delete the expired volume. Deployment ############################# From c9b0cec5b890e6c080c0c05061efa68117e68af1 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Sun, 21 Dec 2014 22:00:59 -0800 Subject: [PATCH 35/67] Include ttl in index --- docs/conf.py | 4 ++-- docs/index.rst | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 8ba3bd55a2..b70feab4e4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,9 +51,9 @@ copyright = u'2014, chrislusf, ernado' # built documents. # # The short X.Y version. -version = '0.63' +version = '0.67' # The full version, including alpha/beta/rc tags. -release = '0.63' +release = '0.67' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/index.rst b/docs/index.rst index 06fdeca7e8..995ba25803 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -27,6 +27,7 @@ Contents: clients api replication + ttl failover usecases directories From 1384fed3ebaf8c93d1568fb97d4d74a17101ece7 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Sun, 21 Dec 2014 22:03:42 -0800 Subject: [PATCH 36/67] Update ttl.rst --- docs/ttl.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/ttl.rst b/docs/ttl.rst index 0d36ec547c..97356e5a74 100644 --- a/docs/ttl.rst +++ b/docs/ttl.rst @@ -12,12 +12,10 @@ How to use it? Assume we want to store a file with TTL of 3 minutes. First, ask the master to assign a file id to a volume with a 3-minute TTL: -.. code-block:: bash > curl http://localhost:9333/dir/assign?ttl=3m {"count":1,"fid":"5,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"} Secondly, use the file id to store on the volume server -.. code-block:: bash > curl -F "file=@x.go" http://127.0.0.1:8080/5,01637037d6?ttl=3m After writing, the file content will be returned as usual if read before the TTL expiry. But if read after the TTL expiry, the file will be reported as missing and return the http response status as not found. From 986797c1c82f6b93cf19973296f98560f222f6fd Mon Sep 17 00:00:00 2001 From: chrislusf Date: Sun, 21 Dec 2014 22:10:53 -0800 Subject: [PATCH 37/67] Adjust wording. --- docs/ttl.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/ttl.rst b/docs/ttl.rst index 97356e5a74..35900b0d8d 100644 --- a/docs/ttl.rst +++ b/docs/ttl.rst @@ -1,4 +1,4 @@ -Store file with a Time To Live (TTL). +Store file with a Time To Live =================== Introduction @@ -12,10 +12,12 @@ How to use it? Assume we want to store a file with TTL of 3 minutes. First, ask the master to assign a file id to a volume with a 3-minute TTL: +.. code-block:: bash > curl http://localhost:9333/dir/assign?ttl=3m {"count":1,"fid":"5,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"} Secondly, use the file id to store on the volume server +.. code-block:: bash > curl -F "file=@x.go" http://127.0.0.1:8080/5,01637037d6?ttl=3m After writing, the file content will be returned as usual if read before the TTL expiry. But if read after the TTL expiry, the file will be reported as missing and return the http response status as not found. @@ -52,9 +54,9 @@ Supported TTL format examples: How efficient it is? ############################# -TTL seems easy to implement since we just need to report the file as missing if the time is over the TTL. However, the real difficulty is to efficiently reclaim disk space from expired files, similar to JVM memory garbage collection, which is a sophisticated piece of work with many man-years of work. +TTL seems easy to implement since we just need to report the file as missing if the time is over the TTL. However, the real difficulty is to efficiently reclaim disk space from expired files, similar to JVM memory garbage collection, which is a sophisticated piece of work with many man-years of effort. -Memcached also supports TTL. It gets around this problem by putting entries into fix-sized slabs. If one slab is expired, no work is required and the slab can be overwritten right away. However, this fix-sized slab approach wastes disk spaces since the file contents rarely fit in slabs exactly. And the slab managements are not trivial either. +Memcached also supports TTL. It gets around this problem by putting entries into fix-sized slabs. If one slab is expired, no work is required and the slab can be overwritten right away. However, this fix-sized slab approach is not applicable to files since the file contents rarely fit in slabs exactly. Seaweed-FS efficiently resolves this disk space garbage collection problem with great simplicity. One of key differences from "normal" implementation is that the TTL is associated with the volume, together with each specific file. From c02a2e25c48a33a12053c3ee658bae6f620baffc Mon Sep 17 00:00:00 2001 From: chrislusf Date: Sun, 21 Dec 2014 22:32:51 -0800 Subject: [PATCH 38/67] Update ttl.rst --- docs/ttl.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/ttl.rst b/docs/ttl.rst index 35900b0d8d..ff579686ba 100644 --- a/docs/ttl.rst +++ b/docs/ttl.rst @@ -12,12 +12,16 @@ How to use it? Assume we want to store a file with TTL of 3 minutes. First, ask the master to assign a file id to a volume with a 3-minute TTL: + .. code-block:: bash + > curl http://localhost:9333/dir/assign?ttl=3m {"count":1,"fid":"5,01637037d6","url":"127.0.0.1:8080","publicUrl":"localhost:8080"} Secondly, use the file id to store on the volume server + .. code-block:: bash + > curl -F "file=@x.go" http://127.0.0.1:8080/5,01637037d6?ttl=3m After writing, the file content will be returned as usual if read before the TTL expiry. But if read after the TTL expiry, the file will be reported as missing and return the http response status as not found. From 33602a03a4fd5b3d2be4f98a961aeab3608e32b1 Mon Sep 17 00:00:00 2001 From: chrislusf Date: Sun, 21 Dec 2014 23:06:42 -0800 Subject: [PATCH 39/67] misc doc changes --- docs/benchmarks.rst | 37 ++++++++++++++++--------------------- docs/clients.rst | 14 +++++++------- docs/directories.rst | 16 +++++++++------- docs/index.rst | 2 +- 4 files changed, 33 insertions(+), 36 deletions(-) diff --git a/docs/benchmarks.rst b/docs/benchmarks.rst index d0b71428fe..6af7ee89ae 100644 --- a/docs/benchmarks.rst +++ b/docs/benchmarks.rst @@ -1,7 +1,10 @@ Benchmarks ====================== -Do we really need the benchmark? People always use benchmark to compare systems. But benchmarks are misleading. The resources, e.g., CPU, disk, memory, network, all matter a lot. And with Seaweed File System, single node vs multiple nodes, benchmarking on one machine vs several multiple machines, all matter a lot. +Do we really need the benchmark? People always use benchmark to compare systems. +But benchmarks are misleading. The resources, e.g., CPU, disk, memory, network, +all matter a lot. And with Seaweed File System, single node vs multiple nodes, +benchmarking on one machine vs several multiple machines, all matter a lot. Here is the steps on how to run benchmark if you really need some numbers. @@ -25,9 +28,13 @@ For more realistic tests, please start them on different machines. What does the test do? ############################# -By default, the benchmark command would start writing 1 million files, each having 1KB size, uncompressed. For each file, one request is sent to assign a file key, and a second request is sent to post the file to the volume server. The written file keys are stored in a temp file. +By default, the benchmark command would start writing 1 million files, each having 1KB size, uncompressed. +For each file, one request is sent to assign a file key, and a second request is sent to post the file to the volume server. +The written file keys are stored in a temp file. -Then the benchmark command would read the list of file keys, randomly read 1 million files. For each volume, the volume id is cached, so there is several request to lookup the volume id, and all the rest requests are to get the file content. +Then the benchmark command would read the list of file keys, randomly read 1 million files. +For each volume, the volume id is cached, so there is several request to lookup the volume id, +and all the rest requests are to get the file content. Many options are options are configurable. Please check the help content: @@ -35,25 +42,11 @@ Many options are options are configurable. Please check the help content: ./weed benchmark -h -Common Problems +Different Benchmark Target ############################### -The most common -I start weed servers in one console for simplicity. Better run servers on different consoles. - -For more realistic tests, please start them on different machines. - -.. code-block:: bash - - # prepare directories - mkdir 3 4 5 - # start 3 servers - ./weed server -dir=./3 -master.port=9333 -volume.port=8083 & - ./weed volume -dir=./4 -port=8084 & - ./weed volume -dir=./5 -port=8085 & - ./weed benchmark -server=localhost:9333 - - problem is "too many open files" error. This is because the test itself starts too many network connections on one single machine. In my local macbook, if I ran "random read" following writing right away, the error happens always. I have to run "weed benchmark -write=false" to run the reading test only. Also, changing the concurrency level to "-c=16" would also help. +The default "weed benchmark" uses 1 million 1KB file. This is to stress the number of files per second. +Increasing the file size to 100KB or more can show much larger number of IO throughput in KB/second. My own unscientific single machine results ################################################### @@ -171,4 +164,6 @@ Create benchmark volumes directly 99% 9.4 ms 100% 256.9 ms How can the replication 001 writes faster than no replication? -I could not tell. Very likely, the computer was in turbo mode. I can not reproduce it consistently either. Posted the number here just to illustrate that number lies. Don't quote on the exact number, just get an idea of the performance would be good enough. \ No newline at end of file +I could not tell. Very likely, the computer was in turbo mode. +I can not reproduce it consistently either. Posted the number here just to illustrate that number lies. +Don't quote on the exact number, just get an idea of the performance would be good enough. \ No newline at end of file diff --git a/docs/clients.rst b/docs/clients.rst index 6c6f097297..e4faacc7e4 100644 --- a/docs/clients.rst +++ b/docs/clients.rst @@ -5,20 +5,20 @@ Clients ################################### +---------------------------------------------------------------------------------+--------------+-----------+ | Name | Author | Language | -+---------------------------------------------------------------------------------+--------------+-----------+ ++=================================================================================+==============+===========+ | `WeedPHP `_ | Mic Johnson | PHP | +---------------------------------------------------------------------------------+--------------+-----------+ -| `Seaweed-FS Symfony bundle `_ | Mic Johnson | PHP | +| `Seaweed-FS Symfony bundle `_ | Mic Johnson | PHP | +---------------------------------------------------------------------------------+--------------+-----------+ -| `Seaweed-FS Node.js client `_ | Aaron Blakely| Javascript| +| `Seaweed-FS Node.js client `_ | Aaron Blakely| Javascript| +---------------------------------------------------------------------------------+--------------+-----------+ -| `Amazon S3 API for Seaweed-FS `_ | Tamás Gulácsi| Go | +| `Amazon S3 API for Seaweed-FS `_ | Tamás Gulácsi| Go | +---------------------------------------------------------------------------------+--------------+-----------+ | `File store upload test `_ | Tamás Gulácsi| Go | +---------------------------------------------------------------------------------+--------------+-----------+ -| `Java Seaweed-FS client `_ | Xu Zhang | Java | +| `Java Seaweed-FS client `_ | Xu Zhang | Java | +---------------------------------------------------------------------------------+--------------+-----------+ -| `Java Seaweed-FS client 2 `_ | Zenria | Java | +| `Java Seaweed-FS client 2 `_ | Zenria | Java | +---------------------------------------------------------------------------------+--------------+-----------+ | `Python-weed `_ | Darkdarkfruit| Python | +---------------------------------------------------------------------------------+--------------+-----------+ @@ -26,7 +26,7 @@ Clients +---------------------------------------------------------------------------------+--------------+-----------+ | `Camlistore blobserver Storage `_ | Tamás Gulácsi| Go | +---------------------------------------------------------------------------------+--------------+-----------+ -| `Scala Seaweed-FS client `_ | Chiradip | Scala | +| `Scala Seaweed-FS client `_ | Chiradip | Scala | +---------------------------------------------------------------------------------+--------------+-----------+ | `Module for kohana `_ | Bububa | PHP | +---------------------------------------------------------------------------------+--------------+-----------+ diff --git a/docs/directories.rst b/docs/directories.rst index aa14da4f72..985d9ba38d 100644 --- a/docs/directories.rst +++ b/docs/directories.rst @@ -39,7 +39,7 @@ A common file system would use inode to store meta data for each folder and file Seaweed-FS wants to make as small number of disk access as possible, yet still be able to store a lot of file metadata. So we need to think very differently. -From a full file path to get to the file content, there are several steps: +We can take the following steps to map a full file path to the actual data block: .. code-block:: bash @@ -48,7 +48,7 @@ From a full file path to get to the file content, there are several steps: file_id => data_block -Because default Seaweed-FS only provides file_id=>data_block mapping, the first 2 steps need to be implemented. +Because default Seaweed-FS only provides file_id=>data_block mapping, only the first 2 steps need to be implemented. There are several data features I noticed: @@ -72,7 +72,7 @@ I believe these are reasonable assumptions: Data structure ################# -This difference lead to the design that the metadata for directories and files should have different data structure. +This assumed differences between directories and files lead to the design that the metadata for directories and files should have different data structure. * Store directories in memory @@ -100,16 +100,18 @@ For file renaming, it's just trivially delete and then add a row in leveldb. Details ######################## -In the current first version, the path_to_file=>file_id mapping is stored with an efficient embedded leveldb. Being embedded, it runs on single machine. So it's not linearly scalable yet. However, it can handle LOTS AND LOTS of files on weed-fs on other servers. Using an external distributed database is possible. Your contribution is welcome! +In the current first version, the path_to_file=>file_id mapping is stored with an efficient embedded leveldb. Being embedded, it runs on single machine. So it's not linearly scalable yet. However, it can handle LOTS AND LOTS of files on Seaweed-FS on other master/volume servers. -The in-memory directory structure can improve on memory efficiency. Current simple map in memory works when the number of directories is less than 1 million, which will use about 500MB memory. But I would highly doubt any common use case would have more than 100 directories. +Switching from the embedded leveldb to an external distributed database is very feasible. Your contribution is welcome! + +The in-memory directory structure can improve on memory efficiency. Current simple map in memory works when the number of directories is less than 1 million, which will use about 500MB memory. But I would expect common use case would have a few, not even more than 100 directories. Use Cases ######################### Clients can assess one "weed filer" via HTTP, list files under a directory, create files via HTTP POST, read files via HTTP POST directly. -Although one "weed filer" can only sits in one machine, you can start multiple "weed filer" on several machines, each "weed filer" instance running in its own collection, having its own namespace, but sharing the same weed-fs. +Although one "weed filer" can only sits in one machine, you can start multiple "weed filer" on several machines, each "weed filer" instance running in its own collection, having its own namespace, but sharing the same Seaweed-FS storage. Future ################### @@ -127,6 +129,6 @@ Later, FUSE or HCFS plugins will be created, to really integrate Seaweed-FS to e Helps Wanted ######################## -This is a big step towards more interesting weed-fs usage and integration with existing systems. +This is a big step towards more interesting Seaweed-FS usage and integration with existing systems. If you can help to refactor and implement other directory meta data, or file meta data storage, please do so. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 995ba25803..9a3e5b32a8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,8 +29,8 @@ Contents: replication ttl failover - usecases directories + usecases optimization benchmarks From 8651f7c7276042332fb3edf545f7a7a3af3e0dec Mon Sep 17 00:00:00 2001 From: chrislusf Date: Sun, 21 Dec 2014 23:27:04 -0800 Subject: [PATCH 40/67] Add change list. --- docs/changelist.rst | 275 ++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 276 insertions(+) create mode 100644 docs/changelist.rst diff --git a/docs/changelist.rst b/docs/changelist.rst new file mode 100644 index 0000000000..c4cfb9c1d9 --- /dev/null +++ b/docs/changelist.rst @@ -0,0 +1,275 @@ +Change List +=================================== + +Introduction +############ +This file contains list of recent changes, important features, usage changes, data format changes, etc. Do read this if you upgrade. + + +v0.67 +##### +1. Increase "weed benchmark" performance to pump in more data. The bottleneck is on the client side. Duh... + +v0.65 +##### + +1. Reset the cluster configuration if "-peers" is not empty. + +v0.64 +##### + +1. Add TTL support! +1. filer: resolve directory log file error, avoid possible race condition + +v0.63 +##### + +1. Compiled with Go 1.3.1 to fix a rare crashing issue. + +v0.62 +##### + +1. Add support for Etag. +2. Add /admin/mv to move a file or a folder. +3. Add client Go API to pre-process the images. + +v0.61 +##### + +1. Reduce memory requirements for "weed fix" +2. Guess mime type by file name extensions when stored mime type is "application/octstream" +3. Added simple volume id lookup caching expiring by time. + +v0.60 +##### + +Fix file missing error caused by .idx file overwriting. The problem shows up if the weed volume server is restarted after 2 times. But the actual .idx file may have already been overwritten on second restart. + +To fix this issue, please run "weed fix -dir=... -volumeId=..." to re-generate the .idx file. + +v0.59 +##### + +1. Add option to automatically fix jpeg picture orientation. +2. Add volume id lookup caching +3. Support Partial Content and Range Requests. http status code == 206. + +v0.57 +##### + +Add hidden dynamic image resizing feature + +Add an hidden feature: For images, jpg/png/gif, if you specify append these url parameters, &width=xxx or &height=xxx or both, the image will be dynamically resized. However, resizing the image would cause high CPU and memory usage. Not recommended unless special use cases. So this would not be documented anywhere else. + +v0.56 Major Command line options change +##### + + +Adjust command line options. + +1. switch to use -publicIp instead of -publicUrl +2. -ip can be empty. It will listen to all available interfaces. +3. For "weed server", these options are changed: + - -masterPort => -master.port + - -peers => -master.peers + - -mdir => -master.dir + - -volumeSizeLimitMB => -master.volumeSizeLimitMB + - -conf => -master.conf + - -defaultReplicaPlacement => -master.defaultReplicaPlacement + - -port => -volume.port + - -max => -volume.max + +v0.55 Recursive folder deletion for Filer +##### + +Now folders with sub folders or files can be deleted recursively. + +Also, for filer, avoid showing files under the first created directory when listing the root directory. + +v0.54 Misc improvements +##### + +No need to persist metadata for master sequence number generation. This shall avoid possible issues where file are lost due to duplicated sequence number generated in rare cases. + +More robust handing of "peers" in master node clustering mode. + +Added logging instructions. + +v0.53 Miscellaneous improvements +##### + +Added retry logic to wait for cluster peers during cluster bootstrapping. Previously the cluster bootstrapping is ordered. This make it tricky to deploy automatically and repeatedly. The fix make the commands repeatable. + +Also, when growing volumes, additional preferred "rack" and "dataNode" parameters are also provided, works together with existing "dataCenter" parameter. + +Fix important bug where settings for non-"000" replications are read back wrong, if volume server is restarted. + +v0.52 Added "filer" server +##### + +A "weed filer" server is added, to provide more "common" file storage. Currently the fullFileName-to-fileId mapping is stored with an efficient embedded leveldb. So it's not linearly scalable yet. But it can handle LOTS of files. + + +.. code-block:: bash + + //POST a file and read it back + curl -F "filename=@README.md" "http://localhost:8888/path/to/sources/" + curl "http://localhost:8888/path/to/sources/README.md" + //POST a file with a new name and read it back + curl -F "filename=@Makefile" "http://localhost:8888/path/to/sources/new_name" + curl "http://localhost:8888/path/to/sources/new_name" + //list sub folders and files + curl "http://localhost:8888/path/to/sources/?pretty=y" + + +v0.51 Idle Timeout +##### + +Previously the timeout setting is "-readTimeout", which is the time limit of the whole http connection. This is inconvenient for large files or for slow internet connections. Now this option is replaced with "-idleTimeout", and default to 10 seconds. Ideally, you should not need to tweak it based on your use case. + +v0.50 Improved Locking +##### + +1. All read operation switched to thread-safe pread, no read locks now. +2. When vacuuming large volumes, a lock was preventing heartbeats to master node. This is fixed now. +3. Fix volume compaction error for collections. + +v0.49 Bug Fixes +##### + +With the new benchmark tool to bombard the system, many bugs are found and fixed, especially on clustering, http connection reuse. + +v0.48 added benchmark command! +##### + +Benchmark! Enough said. + +v0.47 Improving replication +##### + +Support more replication types. + +v0.46 Adding failover master server +##### + +Automatically fail over master servers! + +v0.46 Add "weed server" command +##### + +Now you can start one master server and one volume server in just one command! + +.. code-block:: bash + + weed server + + +v0.45 Add support for extra large file +##### + +For extra large file, this example will split the file into 100MB chunks. + +.. code-block:: bash + + weed upload -maxMB=100 the_file_name + + +Also, Added "download" command, for simple files or chunked files. + +.. code-block:: bash + + weed download file_id [file_id3](file_id2) + + +v0.34 Add support for multiple directories on volume server +##### + +For volume server, add support for multiple folders and multiple max limit. For example: + +.. code-block:: bash + + weed volume -dir=folder1,folder2,folder3 -max=7,8,9 + + +v0.33 Add Nicer URL support +##### + +For HTTP GET request + +.. code-block:: bash + + http://localhost:8080/3,01637037d6 + +Can also be retrieved by + +.. code-block:: bash + + http://localhost:8080/3/01637037d6/my_preferred_name.jpg + + +v0.32 Add support for Last-Modified header +##### + +The last modified timestamp is stored with 5 additional bytes. + +Return http code 304 if the file is not modified. + +Also, the writing are more solid with the fix for issue#26. + +v0.31 Allocate File Key on specific data center +##### + +Volume servers can start with a specific data center name. + +.. code-block:: bash + + weed volume -dir=/tmp/1 -port=8080 -dataCenter=dc1 + weed volume -dir=/tmp/2 -port=8081 -dataCenter=dc2 + +Or the master server can determine the data center via volume server's IP address and settings in weed.conf file. + +Now when requesting a file key, an optional "dataCenter" parameter can limit the assigned volume to the specific data center. For example, this specif + +.. code-block:: bash + + http://localhost:9333/dir/assign?dataCenter=dc1 + +v0.26 Storing File Name and Mime Type +##### + +In order to keep one single disk read for each file, a new storage format is implemented to store: is gzipped or not, file name and mime type (used when downloading files), and possibly other future new attributes. The volumes with old storage format are treated as read only and deprecated. + +Also, you can pre-gzip and submit your file directly, for example, gzip "my.css" into "my.css.gz", and submit. In this case, "my.css" will be stored as the file name. This should save some transmission time, and allow you to force gzipped storage or customize the gzip compression level. + +v0.25 Adding reclaiming garbage spaces + +Garbage spaces are reclaimed by an automatic compacting process. Garbage spaces are generated when updating or deleting files. If they exceed a configurable threshold, 0.3 by default (meaning 30% of the used disk space is garbage), the volume will be marked as readonly, compacted and garbage spaces are reclaimed, and then marked as writable. + +v0.19 Adding rack and data center aware replication +##### + +Now when you have one rack, or multiple racks, or multiple data centers, you can choose your own replication strategy. + +v0.18 Detect disconnected volume servers +##### + +The disconnected volume servers would not be assigned when generating the file keys. Volume servers by default send a heartbeat to master server every 5~10 seconds. Master thinks the volume server is disconnected after 5 times of the heartbeat interval, or 25 seconds by default. + +v0.16 Change to single executable file to do everything +##### + +If you are using v0.15 or earlier, you would use + +.. code-block:: bash + + >weedvolume -dir="/tmp" -volumes=0-4 -mserver="localhost:9333" -port=8080 -publicUrl="localhost:8080" + +With v0.16 or later, you would need to do this in stead: + +.. code-block:: bash + + >weed volume -dir="/tmp" -volumes=0-4 -mserver="localhost:9333" -port=8080 -publicUrl="localhost:8080" + +And more new commands, in addition to "server","volume","fix", etc, will be added. + +This provides a simple deliverable file, and the file size is much smaller since Go language statically compile the commands. Combining commands into one file would avoid lots of duplication. diff --git a/docs/index.rst b/docs/index.rst index 9a3e5b32a8..bf696558a1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -33,6 +33,7 @@ Contents: usecases optimization benchmarks + changelist Indices and tables From 089eb8ad3900d6e61343d6606f094a2ffe9b0a9e Mon Sep 17 00:00:00 2001 From: yanyiwu Date: Fri, 26 Dec 2014 13:29:44 +0800 Subject: [PATCH 41/67] add String function for needle to print --- go/storage/needle.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/go/storage/needle.go b/go/storage/needle.go index aa3206920e..5b43576148 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -3,6 +3,7 @@ package storage import ( "encoding/hex" "errors" + "fmt" "io/ioutil" "mime" "net/http" @@ -45,6 +46,11 @@ type Needle struct { Padding []byte `comment:"Aligned to 8 bytes"` } +func (n *Needle) String() (str string) { + str = fmt.Sprintf("Cookie:%d, Id:%d, Size:%d, DataSize:%d, Name: %s, Mime: %s", n.Cookie, n.Id, n.Size, n.DataSize, n.Name, n.Mime) + return +} + func ParseUpload(r *http.Request) (fileName string, data []byte, mimeType string, isGzipped bool, modifiedTime uint64, ttl *TTL, e error) { form, fe := r.MultipartReader() if fe != nil { From 5a40f539f23bbc262c3a10f64cc53bba67b3df21 Mon Sep 17 00:00:00 2001 From: yanyiwu Date: Fri, 26 Dec 2014 15:36:33 +0800 Subject: [PATCH 42/67] fix bug: upload a file which already existed return a wrong file size. --- go/storage/needle.go | 1 + go/storage/store.go | 12 ++++++++++++ go/storage/volume.go | 8 ++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/go/storage/needle.go b/go/storage/needle.go index 5b43576148..11610dd802 100644 --- a/go/storage/needle.go +++ b/go/storage/needle.go @@ -25,6 +25,7 @@ const ( ) /* +* A Needle means a uploaded and stored file. * Needle file size is limited to 4GB for now. */ type Needle struct { diff --git a/go/storage/store.go b/go/storage/store.go index 80d8a30b81..e1d9d2b018 100644 --- a/go/storage/store.go +++ b/go/storage/store.go @@ -33,6 +33,10 @@ type MasterNodes struct { lastNode int } +func (mn *MasterNodes) String() string { + return fmt.Sprintf("nodes:%v, lastNode:%d", mn.nodes, mn.lastNode) +} + func NewMasterNodes(bootstrapNode string) (mn *MasterNodes) { mn = &MasterNodes{nodes: []string{bootstrapNode}, lastNode: -1} return @@ -65,6 +69,9 @@ func (mn *MasterNodes) findMaster() (string, error) { return mn.nodes[mn.lastNode], nil } +/* + * A VolumeServer contains one Store + */ type Store struct { Port int Ip string @@ -77,6 +84,11 @@ type Store struct { masterNodes *MasterNodes } +func (s *Store) String() (str string) { + str = fmt.Sprintf("Ip:%s, Port:%d, PublicUrl:%s, dataCenter:%s, rack:%s, connected:%v, volumeSizeLimit:%d, masterNodes:%s", s.Ip, s.Port, s.PublicUrl, s.dataCenter, s.rack, s.connected, s.volumeSizeLimit, s.masterNodes) + return +} + func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int) (s *Store) { s = &Store{Port: port, Ip: ip, PublicUrl: publicUrl} s.Locations = make([]*DiskLocation, 0) diff --git a/go/storage/volume.go b/go/storage/volume.go index 5b0a836054..a1eccd62ca 100644 --- a/go/storage/volume.go +++ b/go/storage/volume.go @@ -33,6 +33,10 @@ func NewVolume(dirname string, collection string, id VolumeId, replicaPlacement e = v.load(true, true) return } +func (v *Volume) String() string { + return fmt.Sprintf("Id:%v, dir:%s, Collection:%s, dataFile:%v, nm:%v, readOnly:%v", v.Id, v.dir, v.Collection, v.dataFile, v.nm, v.readOnly) +} + func loadVolumeWithoutIndex(dirname string, collection string, id VolumeId) (v *Volume, e error) { v = &Volume{dir: dirname, Collection: collection, Id: id} v.SuperBlock = SuperBlock{} @@ -135,7 +139,7 @@ func (v *Volume) isFileUnchanged(n *Needle) bool { oldNeedle := new(Needle) oldNeedle.Read(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version()) if oldNeedle.Checksum == n.Checksum && bytes.Equal(oldNeedle.Data, n.Data) { - n.Size = oldNeedle.Size + n.DataSize = oldNeedle.DataSize return true } } @@ -165,7 +169,7 @@ func (v *Volume) write(n *Needle) (size uint32, err error) { v.accessLock.Lock() defer v.accessLock.Unlock() if v.isFileUnchanged(n) { - size = n.Size + size = n.DataSize glog.V(4).Infof("needle is unchanged!") return } From 5b7628cf08b246bb8dd075a28f29e1bd3401eb2d Mon Sep 17 00:00:00 2001 From: yanyiwu Date: Fri, 26 Dec 2014 16:59:53 +0800 Subject: [PATCH 43/67] use github.com/golang/protobuf/proto instead of code.google.com/p/goprotobuf/proto --- go/operation/system_message.pb.go | 2 +- go/operation/system_message_test.go | 2 +- go/storage/store.go | 2 +- go/weed/weed_server/master_server_handlers_admin.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/operation/system_message.pb.go b/go/operation/system_message.pb.go index 9f00dd74d8..6f0f974c5a 100644 --- a/go/operation/system_message.pb.go +++ b/go/operation/system_message.pb.go @@ -14,7 +14,7 @@ It has these top-level messages: */ package operation -import proto "code.google.com/p/goprotobuf/proto" +import "github.com/golang/protobuf/proto" import math "math" // Reference imports to suppress errors if they are not otherwise used. diff --git a/go/operation/system_message_test.go b/go/operation/system_message_test.go index b5624c2588..d18ca49a45 100644 --- a/go/operation/system_message_test.go +++ b/go/operation/system_message_test.go @@ -5,7 +5,7 @@ import ( "log" "testing" - proto "code.google.com/p/goprotobuf/proto" + "github.com/golang/protobuf/proto" ) func TestSerialDeserial(t *testing.T) { diff --git a/go/storage/store.go b/go/storage/store.go index e1d9d2b018..65eed1d0e6 100644 --- a/go/storage/store.go +++ b/go/storage/store.go @@ -9,10 +9,10 @@ import ( "strconv" "strings" - proto "code.google.com/p/goprotobuf/proto" "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/operation" "github.com/chrislusf/weed-fs/go/util" + "github.com/golang/protobuf/proto" ) const ( diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index d7124e5676..75e0593658 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -8,12 +8,12 @@ import ( "strconv" "strings" - proto "code.google.com/p/goprotobuf/proto" "github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/operation" "github.com/chrislusf/weed-fs/go/storage" "github.com/chrislusf/weed-fs/go/topology" "github.com/chrislusf/weed-fs/go/util" + "github.com/golang/protobuf/proto" ) func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) { From e778d047401253451abe7480a37f0f3a3c1aa224 Mon Sep 17 00:00:00 2001 From: Sam Ng Date: Thu, 1 Jan 2015 14:25:59 -0700 Subject: [PATCH 44/67] An extremely small 19.6 MB docker image for weed-fs --- Dockerfile | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 06d81ebf0b..00b9580466 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,21 @@ -FROM cydev/go -RUN go get github.com/chrislusf/weed-fs/go/weed +FROM progrium/busybox + +WORKDIR /opt/weed + +RUN opkg-install curl +RUN echo insecure >> ~/.curlrc + +RUN \ + curl -Lks https://bintray.com$(curl -Lk http://bintray.com/chrislusf/Weed-FS/seaweed/_latestVersion | grep linux_amd64.tar.gz | sed -n "/href/ s/.*href=['\"]\([^'\"]*\)['\"].*/\1/gp") | gunzip | tar -xf - -C /opt/weed/ && \ + mv weed_* bin && \ + chmod +x ./bin/weed + EXPOSE 8080 EXPOSE 9333 + VOLUME /data -ENTRYPOINT ["weed"] + +ENV WEED_HOME /opt/weed +ENV PATH ${PATH}:${WEED_HOME}/bin + +ENTRYPOINT ["weed"] \ No newline at end of file From 329c5915fe18a2fd48de5512a51cec31d029ad64 Mon Sep 17 00:00:00 2001 From: Sam Ng Date: Thu, 1 Jan 2015 18:56:51 -0700 Subject: [PATCH 45/67] Go Build Dockerfile --- Dockerfile.go_build | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 Dockerfile.go_build diff --git a/Dockerfile.go_build b/Dockerfile.go_build new file mode 100644 index 0000000000..8463007d1c --- /dev/null +++ b/Dockerfile.go_build @@ -0,0 +1,6 @@ +FROM cydev/go +RUN go get github.com/chrislusf/weed-fs/go/weed +EXPOSE 8080 +EXPOSE 9333 +VOLUME /data +ENTRYPOINT ["weed"] \ No newline at end of file From b51282ae95ade0e77eecb38b3f18246a7800d014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=96=E3=83=89=E3=82=A6=E3=81=AE=E9=B3=A5?= Date: Thu, 1 Jan 2015 20:59:56 -0700 Subject: [PATCH 46/67] Update gettingstarted.rst Adding instructions to build docker image --- docs/gettingstarted.rst | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst index ad28319fec..21672b811c 100644 --- a/docs/gettingstarted.rst +++ b/docs/gettingstarted.rst @@ -82,6 +82,9 @@ Using Seaweed-FS in docker You can use image "cydev/weed" or build your own with `dockerfile `_ in the root of repo. +Using pre-built Docker image +************************************************************** + .. code-block:: bash docker run --name weed cydev/weed server @@ -98,6 +101,29 @@ And in another terminal } # use $IP as host for api queries +Building image from dockerfile +************************************************************** + +Make a local copy of weed-fs from github + +.. code-block:: bash + + git clone https://github.com/chrislusf/weed-fs.git + +Minimal Image (~19.6 MB) + +.. code-block:: bash + + docker build --no-cache --rm -t 'cydev/weed' . + +Go-Build Docker Image (~764 MB) + +.. code-block:: bash + + mv Dockerfile Dockerfile.minimal + mv Dockerfile.go_build Dockerfile + docker build --no-cache --rm -t 'cydev/weed' . + In production ************************************************************** @@ -111,4 +137,4 @@ To gain persistency you can use docker volumes. -publicIp="$(curl -s cydev.ru/ip)" Now our weed-fs server will be persistent and accessible by localhost:9333 and :8080 on host machine. -Dont forget to specify "-publicIp" for correct connectivity. \ No newline at end of file +Dont forget to specify "-publicIp" for correct connectivity. From 3d3a81048828ed5133530f989478569ab3259d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=83=96=E3=83=89=E3=82=A6=E3=81=AE=E9=B3=A5?= Date: Thu, 1 Jan 2015 21:52:42 -0700 Subject: [PATCH 47/67] Update gettingstarted.rst --- docs/gettingstarted.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst index 21672b811c..e0ca6f94d6 100644 --- a/docs/gettingstarted.rst +++ b/docs/gettingstarted.rst @@ -114,7 +114,7 @@ Minimal Image (~19.6 MB) .. code-block:: bash - docker build --no-cache --rm -t 'cydev/weed' . + docker build --no-cache -t 'cydev/weed' . Go-Build Docker Image (~764 MB) @@ -122,7 +122,7 @@ Go-Build Docker Image (~764 MB) mv Dockerfile Dockerfile.minimal mv Dockerfile.go_build Dockerfile - docker build --no-cache --rm -t 'cydev/weed' . + docker build --no-cache -t 'cydev/weed' . In production ************************************************************** From a3e4145e8ab8958e750459dd001a24d5b5f514a3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 5 Jan 2015 14:20:04 -0800 Subject: [PATCH 48/67] refactoring for later security changes --- go/security/guard.go | 146 ++++++++++++++++++ go/weed/master.go | 4 +- go/weed/server.go | 8 +- go/weed/volume.go | 4 +- go/weed/weed_server/common.go | 20 --- go/weed/weed_server/master_server.go | 32 ++-- go/weed/weed_server/volume_server.go | 31 ++-- go/weed/weed_server/volume_server_handlers.go | 6 +- note/security.txt | 36 +++++ 9 files changed, 230 insertions(+), 57 deletions(-) create mode 100644 go/security/guard.go create mode 100644 note/security.txt diff --git a/go/security/guard.go b/go/security/guard.go new file mode 100644 index 0000000000..a2beb48f49 --- /dev/null +++ b/go/security/guard.go @@ -0,0 +1,146 @@ +package security + +import ( + "errors" + "fmt" + "net" + "net/http" + "strings" + "time" + + "github.com/chrislusf/weed-fs/go/glog" + "github.com/dgrijalva/jwt-go" +) + +var ( + ErrUnauthorized = errors.New("unauthorized token") +) + +/* +Guard is to ensure data access security. +There are 2 ways to check access: +1. white list. It's checking request ip address. +2. JSON Web Token(JWT) generated from secretKey. + The jwt can come from: + 1. url parameter jwt=... + 2. request header "Authorization" + 3. cookie with the name "jwt" + +The white list is checked first because it is easy. +Then the JWT is checked. + +The Guard will also check these claims if provided: +1. "exp" Expiration Time +2. "nbf" Not Before + +Generating JWT: +1. use HS256 to sign +2. optionally set "exp", "nbf" fields, in Unix time, + the number of seconds elapsed since January 1, 1970 UTC. + +Referenced: +https://github.com/pkieltyka/jwtauth/blob/master/jwtauth.go + +*/ +type Guard struct { + whiteList []string + secretKey string + + isActive bool +} + +func NewGuard(whiteList []string, secretKey string) *Guard { + g := &Guard{whiteList: whiteList, secretKey: secretKey} + g.isActive = len(g.whiteList) != 0 || len(g.secretKey) != 0 + return g +} + +func (g *Guard) Secure(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { + if !g.isActive { + //if no security needed, just skip all checkings + return f + } + return func(w http.ResponseWriter, r *http.Request) { + if err := g.doCheck(w, r); err != nil { + w.WriteHeader(http.StatusUnauthorized) + return + } + f(w, r) + } +} + +func (g *Guard) NewToken() (tokenString string, err error) { + m := make(map[string]interface{}) + m["exp"] = time.Now().Unix() + 10 + return g.Encode(m) +} + +func (g *Guard) Encode(claims map[string]interface{}) (tokenString string, err error) { + if !g.isActive { + return "", nil + } + + t := jwt.New(jwt.GetSigningMethod("HS256")) + t.Claims = claims + return t.SignedString(g.secretKey) +} + +func (g *Guard) Decode(tokenString string) (token *jwt.Token, err error) { + return jwt.Parse(tokenString, func(token *jwt.Token) (interface{}, error) { + return g.secretKey, nil + }) +} + +func (g *Guard) doCheck(w http.ResponseWriter, r *http.Request) error { + if len(g.whiteList) != 0 { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err == nil { + for _, ip := range g.whiteList { + if ip == host { + return nil + } + } + } + } + + if len(g.secretKey) != 0 { + + // Get token from query params + tokenStr := r.URL.Query().Get("jwt") + + // Get token from authorization header + if tokenStr == "" { + bearer := r.Header.Get("Authorization") + if len(bearer) > 7 && strings.ToUpper(bearer[0:6]) == "BEARER" { + tokenStr = bearer[7:] + } + } + + // Get token from cookie + if tokenStr == "" { + cookie, err := r.Cookie("jwt") + if err == nil { + tokenStr = cookie.Value + } + } + + if tokenStr == "" { + return ErrUnauthorized + } + + // Verify the token + token, err := g.Decode(tokenStr) + if err != nil { + glog.V(1).Infof("Token verification error from %s: %v", r.RemoteAddr, err) + return ErrUnauthorized + } + if !token.Valid { + glog.V(1).Infof("Token invliad from %s: %v", r.RemoteAddr, tokenStr) + return ErrUnauthorized + } + + } + + glog.V(1).Infof("No permission from %s", r.RemoteAddr) + return fmt.Errorf("No write permisson from %s", r.RemoteAddr) +} diff --git a/go/weed/master.go b/go/weed/master.go index f88964b6dd..de4b5cb4bd 100644 --- a/go/weed/master.go +++ b/go/weed/master.go @@ -42,6 +42,7 @@ var ( mMaxCpu = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") garbageThreshold = cmdMaster.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces") masterWhiteListOption = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") + masterSecureKey = cmdMaster.Flag.String("secure.key", "", "secret key to check permission") masterWhiteList []string ) @@ -60,7 +61,8 @@ func runMaster(cmd *Command, args []string) bool { r := mux.NewRouter() ms := weed_server.NewMasterServer(r, *mport, *metaFolder, - *volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold, masterWhiteList, + *volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold, + masterWhiteList, *masterSecureKey, ) listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport) diff --git a/go/weed/server.go b/go/weed/server.go index 67bbdb3708..2db2519445 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -56,6 +56,7 @@ var ( serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name") serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") serverPeers = cmdServer.Flag.String("master.peers", "", "other master nodes in comma separated ip:masterPort list") + serverSecureKey = cmdServer.Flag.String("secure.key", "", "secret key to ensure authenticated access") serverGarbageThreshold = cmdServer.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces") masterPort = cmdServer.Flag.Int("master.port", 9333, "master server http listen port") masterMetaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified") @@ -185,7 +186,8 @@ func runServer(cmd *Command, args []string) bool { go func() { r := mux.NewRouter() ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder, - *masterVolumeSizeLimitMB, *volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *serverGarbageThreshold, serverWhiteList, + *masterVolumeSizeLimitMB, *volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *serverGarbageThreshold, + serverWhiteList, *serverSecureKey, ) glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort)) @@ -217,8 +219,8 @@ func runServer(cmd *Command, args []string) bool { time.Sleep(100 * time.Millisecond) r := http.NewServeMux() volumeServer := weed_server.NewVolumeServer(r, *serverIp, *volumePort, *serverPublicIp, folders, maxCounts, - *serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack, serverWhiteList, - *volumeFixJpgOrientation, + *serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack, + serverWhiteList, *volumeFixJpgOrientation, ) glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort)) diff --git a/go/weed/volume.go b/go/weed/volume.go index 212cb4b33a..22b50cd346 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -27,6 +27,7 @@ var cmdVolume = &Command{ var ( vport = cmdVolume.Flag.Int("port", 8080, "http listen port") + volumeSecurePort = cmdVolume.Flag.Int("port.secure", 8443, "https listen port, active when SSL certs are specified. Not ready yet.") volumeFolders = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...") maxVolumeCounts = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...") ip = cmdVolume.Flag.String("ip", "", "ip or server name") @@ -82,7 +83,8 @@ func runVolume(cmd *Command, args []string) bool { r := http.NewServeMux() volumeServer := weed_server.NewVolumeServer(r, *ip, *vport, *publicIp, folders, maxCounts, - *masterNode, *vpulse, *dataCenter, *rack, volumeWhiteList, + *masterNode, *vpulse, *dataCenter, *rack, + volumeWhiteList, *fixJpgOrientation, ) diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index 39b0830e33..5c782c091a 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "net" "net/http" "path/filepath" "strconv" @@ -72,25 +71,6 @@ func debug(params ...interface{}) { glog.V(4).Infoln(params) } -func secure(whiteList []string, f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - if len(whiteList) == 0 { - f(w, r) - return - } - host, _, err := net.SplitHostPort(r.RemoteAddr) - if err == nil { - for _, ip := range whiteList { - if ip == host { - f(w, r) - return - } - } - } - writeJsonQuiet(w, r, map[string]interface{}{"error": "No write permisson from " + host}) - } -} - func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl string) { m := make(map[string]interface{}) if r.Method != "POST" { diff --git a/go/weed/weed_server/master_server.go b/go/weed/weed_server/master_server.go index d000cb6107..0a4ca5d5ea 100644 --- a/go/weed/weed_server/master_server.go +++ b/go/weed/weed_server/master_server.go @@ -7,6 +7,7 @@ import ( "sync" "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/security" "github.com/chrislusf/weed-fs/go/sequence" "github.com/chrislusf/weed-fs/go/topology" "github.com/chrislusf/weed-fs/go/util" @@ -21,7 +22,6 @@ type MasterServer struct { pulseSeconds int defaultReplicaPlacement string garbageThreshold string - whiteList []string Topo *topology.Topology vg *topology.VolumeGrowth @@ -37,6 +37,7 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string, defaultReplicaPlacement string, garbageThreshold string, whiteList []string, + secureKey string, ) *MasterServer { ms := &MasterServer{ port: port, @@ -44,7 +45,6 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string, pulseSeconds: pulseSeconds, defaultReplicaPlacement: defaultReplicaPlacement, garbageThreshold: garbageThreshold, - whiteList: whiteList, } ms.bounedLeaderChan = make(chan int, 16) seq := sequence.NewMemorySequencer() @@ -56,20 +56,22 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string, ms.vg = topology.NewDefaultVolumeGrowth() glog.V(0).Infoln("Volume Size Limit is", volumeSizeLimitMB, "MB") - r.HandleFunc("/dir/assign", ms.proxyToLeader(secure(ms.whiteList, ms.dirAssignHandler))) - r.HandleFunc("/dir/lookup", ms.proxyToLeader(secure(ms.whiteList, ms.dirLookupHandler))) - r.HandleFunc("/dir/join", ms.proxyToLeader(secure(ms.whiteList, ms.dirJoinHandler))) - r.HandleFunc("/dir/status", ms.proxyToLeader(secure(ms.whiteList, ms.dirStatusHandler))) - r.HandleFunc("/col/delete", ms.proxyToLeader(secure(ms.whiteList, ms.collectionDeleteHandler))) - r.HandleFunc("/vol/lookup", ms.proxyToLeader(secure(ms.whiteList, ms.volumeLookupHandler))) - r.HandleFunc("/vol/grow", ms.proxyToLeader(secure(ms.whiteList, ms.volumeGrowHandler))) - r.HandleFunc("/vol/status", ms.proxyToLeader(secure(ms.whiteList, ms.volumeStatusHandler))) - r.HandleFunc("/vol/vacuum", ms.proxyToLeader(secure(ms.whiteList, ms.volumeVacuumHandler))) - r.HandleFunc("/submit", secure(ms.whiteList, ms.submitFromMasterServerHandler)) - r.HandleFunc("/delete", secure(ms.whiteList, ms.deleteFromMasterServerHandler)) + guard := security.NewGuard(whiteList, secureKey) + + r.HandleFunc("/dir/assign", ms.proxyToLeader(guard.Secure(ms.dirAssignHandler))) + r.HandleFunc("/dir/lookup", ms.proxyToLeader(guard.Secure(ms.dirLookupHandler))) + r.HandleFunc("/dir/join", ms.proxyToLeader(guard.Secure(ms.dirJoinHandler))) + r.HandleFunc("/dir/status", ms.proxyToLeader(guard.Secure(ms.dirStatusHandler))) + r.HandleFunc("/col/delete", ms.proxyToLeader(guard.Secure(ms.collectionDeleteHandler))) + r.HandleFunc("/vol/lookup", ms.proxyToLeader(guard.Secure(ms.volumeLookupHandler))) + r.HandleFunc("/vol/grow", ms.proxyToLeader(guard.Secure(ms.volumeGrowHandler))) + r.HandleFunc("/vol/status", ms.proxyToLeader(guard.Secure(ms.volumeStatusHandler))) + r.HandleFunc("/vol/vacuum", ms.proxyToLeader(guard.Secure(ms.volumeVacuumHandler))) + r.HandleFunc("/submit", guard.Secure(ms.submitFromMasterServerHandler)) + r.HandleFunc("/delete", guard.Secure(ms.deleteFromMasterServerHandler)) r.HandleFunc("/{fileId}", ms.redirectHandler) - r.HandleFunc("/stats/counter", secure(ms.whiteList, statsCounterHandler)) - r.HandleFunc("/stats/memory", secure(ms.whiteList, statsMemoryHandler)) + r.HandleFunc("/stats/counter", guard.Secure(statsCounterHandler)) + r.HandleFunc("/stats/memory", guard.Secure(statsMemoryHandler)) ms.Topo.StartRefreshWritableVolumes(garbageThreshold) diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go index 0a65fd2f67..9ceeb0149b 100644 --- a/go/weed/weed_server/volume_server.go +++ b/go/weed/weed_server/volume_server.go @@ -7,6 +7,7 @@ import ( "time" "github.com/chrislusf/weed-fs/go/glog" + "github.com/chrislusf/weed-fs/go/security" "github.com/chrislusf/weed-fs/go/storage" ) @@ -15,8 +16,8 @@ type VolumeServer struct { pulseSeconds int dataCenter string rack string - whiteList []string store *storage.Store + guard *security.Guard FixJpgOrientation bool } @@ -24,29 +25,31 @@ type VolumeServer struct { func NewVolumeServer(r *http.ServeMux, ip string, port int, publicIp string, folders []string, maxCounts []int, masterNode string, pulseSeconds int, dataCenter string, rack string, - whiteList []string, fixJpgOrientation bool) *VolumeServer { + whiteList []string, + fixJpgOrientation bool) *VolumeServer { publicUrl := publicIp + ":" + strconv.Itoa(port) vs := &VolumeServer{ masterNode: masterNode, pulseSeconds: pulseSeconds, dataCenter: dataCenter, rack: rack, - whiteList: whiteList, FixJpgOrientation: fixJpgOrientation, } vs.store = storage.NewStore(port, ip, publicUrl, folders, maxCounts) - r.HandleFunc("/status", secure(vs.whiteList, vs.statusHandler)) - r.HandleFunc("/admin/assign_volume", secure(vs.whiteList, vs.assignVolumeHandler)) - r.HandleFunc("/admin/vacuum_volume_check", secure(vs.whiteList, vs.vacuumVolumeCheckHandler)) - r.HandleFunc("/admin/vacuum_volume_compact", secure(vs.whiteList, vs.vacuumVolumeCompactHandler)) - r.HandleFunc("/admin/vacuum_volume_commit", secure(vs.whiteList, vs.vacuumVolumeCommitHandler)) - r.HandleFunc("/admin/freeze_volume", secure(vs.whiteList, vs.freezeVolumeHandler)) - r.HandleFunc("/admin/delete_collection", secure(vs.whiteList, vs.deleteCollectionHandler)) - r.HandleFunc("/stats/counter", secure(vs.whiteList, statsCounterHandler)) - r.HandleFunc("/stats/memory", secure(vs.whiteList, statsMemoryHandler)) - r.HandleFunc("/stats/disk", secure(vs.whiteList, vs.statsDiskHandler)) - r.HandleFunc("/delete", secure(vs.whiteList, vs.batchDeleteHandler)) + vs.guard = security.NewGuard(whiteList, "") + + r.HandleFunc("/status", vs.guard.Secure(vs.statusHandler)) + r.HandleFunc("/admin/assign_volume", vs.guard.Secure(vs.assignVolumeHandler)) + r.HandleFunc("/admin/vacuum_volume_check", vs.guard.Secure(vs.vacuumVolumeCheckHandler)) + r.HandleFunc("/admin/vacuum_volume_compact", vs.guard.Secure(vs.vacuumVolumeCompactHandler)) + r.HandleFunc("/admin/vacuum_volume_commit", vs.guard.Secure(vs.vacuumVolumeCommitHandler)) + r.HandleFunc("/admin/freeze_volume", vs.guard.Secure(vs.freezeVolumeHandler)) + r.HandleFunc("/admin/delete_collection", vs.guard.Secure(vs.deleteCollectionHandler)) + r.HandleFunc("/stats/counter", vs.guard.Secure(statsCounterHandler)) + r.HandleFunc("/stats/memory", vs.guard.Secure(statsMemoryHandler)) + r.HandleFunc("/stats/disk", vs.guard.Secure(vs.statsDiskHandler)) + r.HandleFunc("/delete", vs.guard.Secure(vs.batchDeleteHandler)) r.HandleFunc("/", vs.storeHandler) go func() { diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index 83f6149410..3abfe96b58 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -29,13 +29,13 @@ func (vs *VolumeServer) storeHandler(w http.ResponseWriter, r *http.Request) { vs.GetOrHeadHandler(w, r) case "DELETE": stats.DeleteRequest() - secure(vs.whiteList, vs.DeleteHandler)(w, r) + vs.guard.Secure(vs.DeleteHandler)(w, r) case "PUT": stats.WriteRequest() - secure(vs.whiteList, vs.PostHandler)(w, r) + vs.guard.Secure(vs.PostHandler)(w, r) case "POST": stats.WriteRequest() - secure(vs.whiteList, vs.PostHandler)(w, r) + vs.guard.Secure(vs.PostHandler)(w, r) } } diff --git a/note/security.txt b/note/security.txt new file mode 100644 index 0000000000..04030a574c --- /dev/null +++ b/note/security.txt @@ -0,0 +1,36 @@ +Design for Seaweed-FS security + +Design Objectives + Security can mean many different things. The original vision is that: if you have one machine lying around + somewhere with some disk space, it should be able to join your file system to contribute some disk space and + network bandwidth. + + To achieve this purpose, the security should be able to: + 1. Secure the inter-server communication. Only real cluster servers can join and communicate. + 2. allow clients to securely write to volume servers + +Non Objective + Multi-tenant support. Avoid filers or clients cross-updating files. + User specific access control. + +Design Architect + master, and volume servers all talk securely via 2-way SSL for admin. + upon joining, master gives its secret key to volume servers. + filer or clients talk to master to get secret key, and use the key to generate JWT to write on volume server. + A side benefit: + a time limited read feature? + 4. volume server needs to expose https ports + +HTTP Connections + clear http + filer~>master, need to get a JWT from master + filer~>volume + 2-way https + master~ssl~>volume + volume~ssl~>master + +file uploading: + when volume server starts, it asks master for the secret key to decode JWT + when filer/clients wants to upload, master generate a JWT + filer~>volume(public port) + master~>volume(public port) From 165734ce11658f34cb2137e7343516b066c3a1f7 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 5 Jan 2015 14:58:30 -0800 Subject: [PATCH 49/67] Refactoring for supporing cassandra as filer meta data store --- go/filer/directory.go | 18 ------------------ go/filer/embedded_filer/directory.go | 15 +++++++++++++++ .../{ => embedded_filer}/directory_in_map.go | 19 ++++++++++--------- .../{ => embedded_filer}/directory_test.go | 2 +- .../{ => embedded_filer}/filer_embedded.go | 9 +++++---- .../{ => embedded_filer}/files_in_leveldb.go | 15 ++++++++------- go/filer/filer.go | 15 ++++++++++++--- go/weed/weed_server/filer_server.go | 3 ++- 8 files changed, 53 insertions(+), 43 deletions(-) delete mode 100644 go/filer/directory.go create mode 100644 go/filer/embedded_filer/directory.go rename go/filer/{ => embedded_filer}/directory_in_map.go (93%) rename go/filer/{ => embedded_filer}/directory_test.go (98%) rename go/filer/{ => embedded_filer}/filer_embedded.go (95%) rename go/filer/{ => embedded_filer}/files_in_leveldb.go (70%) diff --git a/go/filer/directory.go b/go/filer/directory.go deleted file mode 100644 index 66d1aeba58..0000000000 --- a/go/filer/directory.go +++ /dev/null @@ -1,18 +0,0 @@ -package filer - -type DirectoryId int32 - -type DirectoryEntry struct { - Name string //dir name without path - Id DirectoryId -} - -type DirectoryManager interface { - FindDirectory(dirPath string) (DirectoryId, error) - ListDirectories(dirPath string) (dirs []DirectoryEntry, err error) - MakeDirectory(currentDirPath string, dirName string) (DirectoryId, error) - MoveUnderDirectory(oldDirPath string, newParentDirPath string) error - DeleteDirectory(dirPath string) error - //functions used by FUSE - FindDirectoryById(DirectoryId, error) -} diff --git a/go/filer/embedded_filer/directory.go b/go/filer/embedded_filer/directory.go new file mode 100644 index 0000000000..8ab3e4affc --- /dev/null +++ b/go/filer/embedded_filer/directory.go @@ -0,0 +1,15 @@ +package embedded_filer + +import ( + "github.com/chrislusf/weed-fs/go/filer" +) + +type DirectoryManager interface { + FindDirectory(dirPath string) (filer.DirectoryId, error) + ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) + MakeDirectory(currentDirPath string, dirName string) (filer.DirectoryId, error) + MoveUnderDirectory(oldDirPath string, newParentDirPath string) error + DeleteDirectory(dirPath string) error + //functions used by FUSE + FindDirectoryById(filer.DirectoryId, error) +} diff --git a/go/filer/directory_in_map.go b/go/filer/embedded_filer/directory_in_map.go similarity index 93% rename from go/filer/directory_in_map.go rename to go/filer/embedded_filer/directory_in_map.go index ee601066c9..a1d0f43bd6 100644 --- a/go/filer/directory_in_map.go +++ b/go/filer/embedded_filer/directory_in_map.go @@ -1,4 +1,4 @@ -package filer +package embedded_filer import ( "bufio" @@ -10,6 +10,7 @@ import ( "strings" "sync" + "github.com/chrislusf/weed-fs/go/filer" "github.com/chrislusf/weed-fs/go/util" ) @@ -19,12 +20,12 @@ type DirectoryEntryInMap struct { Name string Parent *DirectoryEntryInMap SubDirectories map[string]*DirectoryEntryInMap - Id DirectoryId + Id filer.DirectoryId } type DirectoryManagerInMap struct { Root *DirectoryEntryInMap - max DirectoryId + max filer.DirectoryId logFile *os.File isLoading bool } @@ -83,7 +84,7 @@ func (dm *DirectoryManagerInMap) processEachLine(line string) error { if pe != nil { return pe } - if e := dm.loadDirectory(parts[1], DirectoryId(v)); e != nil { + if e := dm.loadDirectory(parts[1], filer.DirectoryId(v)); e != nil { return e } case "mov": @@ -142,7 +143,7 @@ func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryI } return dir, nil } -func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (DirectoryId, error) { +func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryId, error) { d, e := dm.findDirectory(dirPath) if e == nil { return d.Id, nil @@ -150,7 +151,7 @@ func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (DirectoryId, err return dm.Root.Id, e } -func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId DirectoryId) error { +func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.DirectoryId) error { dirPath = filepath.Clean(dirPath) if dirPath == "/" { return nil @@ -201,7 +202,7 @@ func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEn return dir, created } -func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (DirectoryId, error) { +func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (filer.DirectoryId, error) { dir, _ := dm.makeDirectory(dirPath) return dir.Id, nil } @@ -228,13 +229,13 @@ func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParent return nil } -func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []DirectoryEntry, err error) { +func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryEntry, err error) { d, e := dm.findDirectory(dirPath) if e != nil { return dirNames, e } for k, v := range d.SubDirectories { - dirNames = append(dirNames, DirectoryEntry{Name: k, Id: v.Id}) + dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id}) } return dirNames, nil } diff --git a/go/filer/directory_test.go b/go/filer/embedded_filer/directory_test.go similarity index 98% rename from go/filer/directory_test.go rename to go/filer/embedded_filer/directory_test.go index ea4903f037..cb0090bf23 100644 --- a/go/filer/directory_test.go +++ b/go/filer/embedded_filer/directory_test.go @@ -1,4 +1,4 @@ -package filer +package embedded_filer import ( "os" diff --git a/go/filer/filer_embedded.go b/go/filer/embedded_filer/filer_embedded.go similarity index 95% rename from go/filer/filer_embedded.go rename to go/filer/embedded_filer/filer_embedded.go index 0b9b4e6687..4321bb4e64 100644 --- a/go/filer/filer_embedded.go +++ b/go/filer/embedded_filer/filer_embedded.go @@ -1,4 +1,4 @@ -package filer +package embedded_filer import ( "errors" @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" + "github.com/chrislusf/weed-fs/go/filer" "github.com/chrislusf/weed-fs/go/operation" ) @@ -48,13 +49,13 @@ func (filer *FilerEmbedded) FindFile(filePath string) (fid string, err error) { } return filer.files.FindFile(dirId, file) } -func (filer *FilerEmbedded) FindDirectory(dirPath string) (dirId DirectoryId, err error) { +func (filer *FilerEmbedded) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) { return filer.directories.FindDirectory(dirPath) } -func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []DirectoryEntry, err error) { +func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) { return filer.directories.ListDirectories(dirPath) } -func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error) { +func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) { dirId, e := filer.directories.FindDirectory(dirPath) if e != nil { return nil, e diff --git a/go/filer/files_in_leveldb.go b/go/filer/embedded_filer/files_in_leveldb.go similarity index 70% rename from go/filer/files_in_leveldb.go rename to go/filer/embedded_filer/files_in_leveldb.go index 781bb0e5f3..c4fab734f3 100644 --- a/go/filer/files_in_leveldb.go +++ b/go/filer/embedded_filer/files_in_leveldb.go @@ -1,8 +1,9 @@ -package filer +package embedded_filer import ( "bytes" + "github.com/chrislusf/weed-fs/go/filer" "github.com/chrislusf/weed-fs/go/glog" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" @@ -27,7 +28,7 @@ func NewFileListInLevelDb(dir string) (fl *FileListInLevelDb, err error) { return } -func genKey(dirId DirectoryId, fileName string) []byte { +func genKey(dirId filer.DirectoryId, fileName string) []byte { ret := make([]byte, 0, 4+len(fileName)) for i := 3; i >= 0; i-- { ret = append(ret, byte(dirId>>(uint(i)*8))) @@ -36,25 +37,25 @@ func genKey(dirId DirectoryId, fileName string) []byte { return ret } -func (fl *FileListInLevelDb) CreateFile(dirId DirectoryId, fileName string, fid string) (err error) { +func (fl *FileListInLevelDb) CreateFile(dirId filer.DirectoryId, fileName string, fid string) (err error) { glog.V(4).Infoln("directory", dirId, "fileName", fileName, "fid", fid) return fl.db.Put(genKey(dirId, fileName), []byte(fid), nil) } -func (fl *FileListInLevelDb) DeleteFile(dirId DirectoryId, fileName string) (fid string, err error) { +func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string) (fid string, err error) { if fid, err = fl.FindFile(dirId, fileName); err != nil { return } err = fl.db.Delete(genKey(dirId, fileName), nil) return fid, err } -func (fl *FileListInLevelDb) FindFile(dirId DirectoryId, fileName string) (fid string, err error) { +func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string) (fid string, err error) { data, e := fl.db.Get(genKey(dirId, fileName), nil) if e != nil { return "", e } return string(data), nil } -func (fl *FileListInLevelDb) ListFiles(dirId DirectoryId, lastFileName string, limit int) (files []FileEntry) { +func (fl *FileListInLevelDb) ListFiles(dirId filer.DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) { glog.V(4).Infoln("directory", dirId, "lastFileName", lastFileName, "limit", limit) dirKey := genKey(dirId, "") iter := fl.db.NewIterator(&util.Range{Start: genKey(dirId, lastFileName)}, nil) @@ -74,7 +75,7 @@ func (fl *FileListInLevelDb) ListFiles(dirId DirectoryId, lastFileName string, l break } } - files = append(files, FileEntry{Name: fileName, Id: FileId(string(iter.Value()))}) + files = append(files, filer.FileEntry{Name: fileName, Id: filer.FileId(string(iter.Value()))}) } iter.Release() return diff --git a/go/filer/filer.go b/go/filer/filer.go index bf4a1cb19c..5a2584c4ac 100644 --- a/go/filer/filer.go +++ b/go/filer/filer.go @@ -7,13 +7,22 @@ type FileEntry struct { Id FileId `json:"fid,omitempty"` } +type DirectoryId int32 + +type DirectoryEntry struct { + Name string //dir name without path + Id DirectoryId +} + type Filer interface { - CreateFile(filePath string, fid string) (err error) - FindFile(filePath string) (fid string, err error) + CreateFile(fullFileName string, fid string) (err error) + FindFile(fullFileName string) (fid string, err error) + DeleteFile(fullFileName string) (fid string, err error) + + //Optional functions. embedded filer support these FindDirectory(dirPath string) (dirId DirectoryId, err error) ListDirectories(dirPath string) (dirs []DirectoryEntry, err error) ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error) DeleteDirectory(dirPath string, recursive bool) (err error) - DeleteFile(filePath string) (fid string, err error) Move(fromPath string, toPath string) (err error) } diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index 0bda58d069..b8cb9bd5d6 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -5,6 +5,7 @@ import ( "strconv" "github.com/chrislusf/weed-fs/go/filer" + "github.com/chrislusf/weed-fs/go/filer/embedded_filer" "github.com/chrislusf/weed-fs/go/glog" ) @@ -28,7 +29,7 @@ func NewFilerServer(r *http.ServeMux, port int, master string, dir string, colle port: ":" + strconv.Itoa(port), } - if fs.filer, err = filer.NewFilerEmbedded(master, dir); err != nil { + if fs.filer, err = embedded_filer.NewFilerEmbedded(master, dir); err != nil { glog.Fatal("Can not start filer in dir", dir, ": ", err.Error()) return } From 49784d7f288d05590ddde5903ef71fe1754da83e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 5 Jan 2015 23:03:27 -0800 Subject: [PATCH 50/67] Add support for distributed filer metadata store. --- go/filer/cassandra_store/cassandra_store.go | 87 +++++++++++++++++++ go/filer/cassandra_store/schema.cql | 22 +++++ go/filer/{ => embedded_filer}/design.txt | 0 .../flat_namespace/flat_namespace_filer.go | 50 +++++++++++ .../flat_namespace/flat_namespace_store.go | 9 ++ go/weed/filer.go | 13 ++- go/weed/server.go | 3 +- go/weed/weed_server/filer_server.go | 22 +++-- go/weed/weed_server/filer_server_handlers.go | 2 +- 9 files changed, 197 insertions(+), 11 deletions(-) create mode 100644 go/filer/cassandra_store/cassandra_store.go create mode 100644 go/filer/cassandra_store/schema.cql rename go/filer/{ => embedded_filer}/design.txt (100%) create mode 100644 go/filer/flat_namespace/flat_namespace_filer.go create mode 100644 go/filer/flat_namespace/flat_namespace_store.go diff --git a/go/filer/cassandra_store/cassandra_store.go b/go/filer/cassandra_store/cassandra_store.go new file mode 100644 index 0000000000..83face686e --- /dev/null +++ b/go/filer/cassandra_store/cassandra_store.go @@ -0,0 +1,87 @@ +package cassandra_store + +import ( + "fmt" + + "github.com/chrislusf/weed-fs/go/glog" + + "github.com/gocql/gocql" +) + +/* + +Basically you need a table just like this: + +CREATE TABLE seaweed_files ( + path varchar, + fids list, + PRIMARY KEY (path) +); + +Need to match flat_namespace.FlatNamespaceStore interface + Put(fullFileName string, fid string) (err error) + Get(fullFileName string) (fid string, err error) + Delete(fullFileName string) (fid string, err error) + +*/ +type CassandraStore struct { + cluster *gocql.ClusterConfig + session *gocql.Session +} + +func NewCassandraStore(keyspace string, hosts ...string) (c *CassandraStore, err error) { + c = &CassandraStore{} + c.cluster = gocql.NewCluster(hosts...) + c.cluster.Keyspace = keyspace + c.cluster.Consistency = gocql.Quorum + c.session, err = c.cluster.CreateSession() + if err != nil { + glog.V(0).Infof("Failed to open cassandra store, hosts %v, keyspace %s", hosts, keyspace) + } + return +} + +func (c *CassandraStore) Put(fullFileName string, fid string) (err error) { + var input []string + input = append(input, fid) + if err := c.session.Query( + `INSERT INTO seaweed_files (path, fids) VALUES (?, ?)`, + fullFileName, input).Exec(); err != nil { + glog.V(0).Infof("Failed to save file %s with id %s: %v", fullFileName, fid, err) + return err + } + return nil +} +func (c *CassandraStore) Get(fullFileName string) (fid string, err error) { + var output []string + if err := c.session.Query( + `select fids FROM seaweed_files WHERE path = ? LIMIT 1`, + fullFileName).Consistency(gocql.One).Scan(&output); err != nil { + if err != gocql.ErrNotFound { + glog.V(0).Infof("Failed to find file %s: %v", fullFileName, fid, err) + } + } + if len(output) == 0 { + return "", fmt.Errorf("No file id found for %s", fullFileName) + } + return output[0], nil +} + +// Currently the fid is not returned +func (c *CassandraStore) Delete(fullFileName string) (fid string, err error) { + if err := c.session.Query( + `DELETE FROM seaweed_files WHERE path = ?`, + fullFileName).Exec(); err != nil { + if err != gocql.ErrNotFound { + glog.V(0).Infof("Failed to delete file %s: %v", fullFileName, err) + } + return "", err + } + return "", nil +} + +func (c *CassandraStore) Close() { + if c.session != nil { + c.session.Close() + } +} diff --git a/go/filer/cassandra_store/schema.cql b/go/filer/cassandra_store/schema.cql new file mode 100644 index 0000000000..d6f2bb0939 --- /dev/null +++ b/go/filer/cassandra_store/schema.cql @@ -0,0 +1,22 @@ +/* + +Here is the CQL to create the table.CassandraStore + +Optionally you can adjust the keyspace name and replication settings. + +For production server, very likely you want to set replication_factor to 3 + +*/ + +create keyspace seaweed WITH replication = { + 'class':'SimpleStrategy', + 'replication_factor':1 +}; + +use seaweed; + +CREATE TABLE seaweed_files ( + path varchar, + fids list, + PRIMARY KEY (path) +); diff --git a/go/filer/design.txt b/go/filer/embedded_filer/design.txt similarity index 100% rename from go/filer/design.txt rename to go/filer/embedded_filer/design.txt diff --git a/go/filer/flat_namespace/flat_namespace_filer.go b/go/filer/flat_namespace/flat_namespace_filer.go new file mode 100644 index 0000000000..021b809d6d --- /dev/null +++ b/go/filer/flat_namespace/flat_namespace_filer.go @@ -0,0 +1,50 @@ +package flat_namespace + +import ( + "errors" + + "github.com/chrislusf/weed-fs/go/filer" +) + +type FlatNamesapceFiler struct { + master string + store FlatNamespaceStore +} + +var ( + NotImplemented = errors.New("Not Implemented for flat namespace meta data store!") +) + +func NewFlatNamesapceFiler(master string, store FlatNamespaceStore) *FlatNamesapceFiler { + return &FlatNamesapceFiler{ + master: master, + store: store, + } +} + +func (filer *FlatNamesapceFiler) CreateFile(fullFileName string, fid string) (err error) { + return filer.store.Put(fullFileName, fid) +} +func (filer *FlatNamesapceFiler) FindFile(fullFileName string) (fid string, err error) { + return filer.store.Get(fullFileName) +} +func (filer *FlatNamesapceFiler) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) { + return 0, NotImplemented +} +func (filer *FlatNamesapceFiler) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) { + return nil, NotImplemented +} +func (filer *FlatNamesapceFiler) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) { + return nil, NotImplemented +} +func (filer *FlatNamesapceFiler) DeleteDirectory(dirPath string, recursive bool) (err error) { + return NotImplemented +} + +func (filer *FlatNamesapceFiler) DeleteFile(fullFileName string) (fid string, err error) { + return filer.store.Delete(fullFileName) +} + +func (filer *FlatNamesapceFiler) Move(fromPath string, toPath string) error { + return NotImplemented +} diff --git a/go/filer/flat_namespace/flat_namespace_store.go b/go/filer/flat_namespace/flat_namespace_store.go new file mode 100644 index 0000000000..832b70e406 --- /dev/null +++ b/go/filer/flat_namespace/flat_namespace_store.go @@ -0,0 +1,9 @@ +package flat_namespace + +import () + +type FlatNamespaceStore interface { + Put(fullFileName string, fid string) (err error) + Get(fullFileName string) (fid string, err error) + Delete(fullFileName string) (fid string, err error) +} diff --git a/go/weed/filer.go b/go/weed/filer.go index 11154f1835..767864450d 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -22,6 +22,8 @@ type FilerOptions struct { defaultReplicaPlacement *string dir *string redirectOnRead *bool + cassandra_server *string + cassandra_keyspace *string } func init() { @@ -32,13 +34,15 @@ func init() { f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data") f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified") f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") + f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server") + f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server") } var cmdFiler = &Command{ UsageLine: "filer -port=8888 -dir=/tmp -master=", Short: "start a file server that points to a master server", Long: `start a file server which accepts REST operation for any files. - + //create or overwrite the file, the directories /path/to will be automatically created POST /path/to/file //get the file content @@ -47,10 +51,10 @@ var cmdFiler = &Command{ POST /path/to/ //return a json format subdirectory and files listing GET /path/to/ - + Current mapping metadata store is local embedded leveldb. It should be highly scalable to hundreds of millions of files on a modest machine. - + Future we will ensure it can avoid of being SPOF. `, @@ -63,8 +67,9 @@ func runFiler(cmd *Command, args []string) bool { } r := http.NewServeMux() - _, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection, + _, nfs_err := weed_server.NewEmbeddedFilerServer(r, *f.port, *f.master, *f.dir, *f.collection, *f.defaultReplicaPlacement, *f.redirectOnRead, + *f.cassandra_server, *f.cassandra_keyspace, ) if nfs_err != nil { glog.Fatalf(nfs_err.Error()) diff --git a/go/weed/server.go b/go/weed/server.go index 2db2519445..3aae2e7fb8 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -157,8 +157,9 @@ func runServer(cmd *Command, args []string) bool { if *isStartingFiler { go func() { r := http.NewServeMux() - _, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection, + _, nfs_err := weed_server.NewEmbeddedFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection, *filerOptions.defaultReplicaPlacement, *filerOptions.redirectOnRead, + "", "", ) if nfs_err != nil { glog.Fatalf(nfs_err.Error()) diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index b8cb9bd5d6..325c9b6267 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -5,7 +5,9 @@ import ( "strconv" "github.com/chrislusf/weed-fs/go/filer" + "github.com/chrislusf/weed-fs/go/filer/cassandra_store" "github.com/chrislusf/weed-fs/go/filer/embedded_filer" + "github.com/chrislusf/weed-fs/go/filer/flat_namespace" "github.com/chrislusf/weed-fs/go/glog" ) @@ -18,8 +20,9 @@ type FilerServer struct { filer filer.Filer } -func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string, +func NewEmbeddedFilerServer(r *http.ServeMux, port int, master string, dir string, collection string, replication string, redirectOnRead bool, + cassandra_server string, cassandra_keyspace string, ) (fs *FilerServer, err error) { fs = &FilerServer{ master: master, @@ -29,12 +32,21 @@ func NewFilerServer(r *http.ServeMux, port int, master string, dir string, colle port: ":" + strconv.Itoa(port), } - if fs.filer, err = embedded_filer.NewFilerEmbedded(master, dir); err != nil { - glog.Fatal("Can not start filer in dir", dir, ": ", err.Error()) - return + if cassandra_server == "" { + if fs.filer, err = embedded_filer.NewFilerEmbedded(master, dir); err != nil { + glog.Fatalf("Can not start filer in dir %s : %v", err) + return + } + + r.HandleFunc("/admin/mv", fs.moveHandler) + } else { + cassandra_store, err := cassandra_store.NewCassandraStore(cassandra_keyspace, cassandra_server) + if err != nil { + glog.Fatalf("Can not connect to cassandra server %s with keyspace %s: %v", cassandra_server, cassandra_keyspace, err) + } + fs.filer = flat_namespace.NewFlatNamesapceFiler(master, cassandra_store) } - r.HandleFunc("/admin/mv", fs.moveHandler) r.HandleFunc("/", fs.filerHandler) return fs, nil diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index 6f22912a79..1afc1f8521 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -179,7 +179,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { glog.V(4).Infoln("saving", path, "=>", assignResult.Fid) if db_err := fs.filer.CreateFile(path, assignResult.Fid); db_err != nil { operation.DeleteFile(fs.master, assignResult.Fid) //clean up - glog.V(0).Infoln("failing to write to filer server", r.RequestURI, db_err.Error()) + glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err) writeJsonError(w, r, db_err) return } From 44f53cba1d7b018848e905c79a08a5e330c6ac02 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 5 Jan 2015 23:06:27 -0800 Subject: [PATCH 51/67] Adjust documents for distributed filer metadata store. --- docs/directories.rst | 18 ++++--- docs/distributed_filer.rst | 101 +++++++++++++++++++++++++++++++++++++ docs/index.rst | 7 +-- 3 files changed, 115 insertions(+), 11 deletions(-) create mode 100644 docs/distributed_filer.rst diff --git a/docs/directories.rst b/docs/directories.rst index 985d9ba38d..92f3646ac0 100644 --- a/docs/directories.rst +++ b/docs/directories.rst @@ -1,19 +1,21 @@ Directories and files =========================== -When talking about file systems, many people would assume directories, list files under a directory, etc. These are expected if we want to hook up Seaweed File System with linux by FUSE, or with Hadoop, etc. +When talking about file systems, many people would assume directories, +list files under a directory, etc. These are expected if we want to hook up +Seaweed File System with linux by FUSE, or with Hadoop, etc. Sample usage ##################### -Two ways to start a weed filer +Two ways to start a weed filer in standalone mode: .. code-block:: bash - + # assuming you already started weed master and weed volume weed filer - # Or assuming you have nothing started yet, - # this command starts master server, volume server, and filer in one shot. + # Or assuming you have nothing started yet, + # this command starts master server, volume server, and filer in one shot. # It's strictly the same as starting them separately. weed server -filer=true @@ -80,10 +82,10 @@ This assumed differences between directories and files lead to the design that t * efficient to move/rename/list_directories * Store files in a sorted string table in format - + * efficient to list_files, just simple iterator * efficient to locate files, binary search - + Complexity ################### @@ -131,4 +133,4 @@ Helps Wanted This is a big step towards more interesting Seaweed-FS usage and integration with existing systems. -If you can help to refactor and implement other directory meta data, or file meta data storage, please do so. \ No newline at end of file +If you can help to refactor and implement other directory meta data, or file meta data storage, please do so. diff --git a/docs/distributed_filer.rst b/docs/distributed_filer.rst new file mode 100644 index 0000000000..2783474815 --- /dev/null +++ b/docs/distributed_filer.rst @@ -0,0 +1,101 @@ +Distributed Filer +=========================== + +The default weed filer is in standalone mode, storing file metadata on disk. +It is quite efficient to go through deep directory path and can handle +millions of files. + +However, no SPOF is a must-have requirement for many projects. + +Luckily, SeaweedFS is so flexible that we can use a completely different way +to manage file metadata. + +This distributed filer uses Cassandra to store the metadata. + +Cassandra Setup +##################### +Here is the CQL to create the table.CassandraStore. +Optionally you can adjust the keyspace name and replication settings. +For production server, you would want to set replication_factor to 3. + +.. code-block:: bash + + create keyspace seaweed WITH replication = { + 'class':'SimpleStrategy', + 'replication_factor':1 + }; + + use seaweed; + + CREATE TABLE seaweed_files ( + path varchar, + fids list, + PRIMARY KEY (path) + ); + + +Sample usage +##################### + +To start a weed filer in distributed mode: + +.. code-block:: bash + + # assuming you already started weed master and weed volume + weed filer -cassandra.server=localhost + +Now you can add/delete files, and even browse the sub directories and files + +.. code-block:: bash + + # POST a file and read it back + curl -F "filename=@README.md" "http://localhost:8888/path/to/sources/" + curl "http://localhost:8888/path/to/sources/README.md" + # POST a file with a new name and read it back + curl -F "filename=@Makefile" "http://localhost:8888/path/to/sources/new_name" + curl "http://localhost:8888/path/to/sources/new_name" + +Limitation +############ +List sub folders and files are not supported because Cassandra does not support +prefix search. + +Flat Namespace Design +############ +In stead of using both directory and file metadata, this implementation uses +a flat namespace. + +If storing each directory metadata separatedly, there would be multiple +network round trips to fetch directory information for deep directories, +impeding system performance. + +A flat namespace would take more space because the parent directories are +repeatedly stored. But disk space is a lesser concern especially for +distributed systems. + +Complexity +################### + +For one file retrieval, the full_filename=>file_id lookup will be O(logN) +using Cassandra. But very likely the one additional network hop would +take longer than the Cassandra internal lookup. + +Use Cases +######################### + +Clients can assess one "weed filer" via HTTP, list files under a directory, create files via HTTP POST, read files via HTTP POST directly. + +Although one "weed filer" can only sits in one machine, you can start multiple "weed filer" on several machines, each "weed filer" instance running in its own collection, having its own namespace, but sharing the same Seaweed-FS storage. + +Future +################### + +The Cassandra implementation can be switched to other distributed hash table. + +Helps Wanted +######################## + +Please implement your preferred metadata store! + +Just follow the cassandra_store/cassandra_store.go file and send me a pull +request. I will handle the rest. diff --git a/docs/index.rst b/docs/index.rst index bf696558a1..429e76d64e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,14 +6,14 @@ Welcome to weed-fs's documentation! =================================== -An official mirror of code.google.com/p/weed-fs . -Moving to github.com to make cooperations easier. +An official mirror of code.google.com/p/weed-fs . +Moving to github.com to make cooperations easier. This repo and the google code repo will be kept synchronized. For documents and bug reporting, Please visit http://weed-fs.googlecode.com - + For pre-compiled releases, https://bintray.com/chrislusf/Weed-FS/seaweed @@ -30,6 +30,7 @@ Contents: ttl failover directories + distributed_filer usecases optimization benchmarks From 8d9a77c9b7e90723a2eedb4a35992d2159fc5bd3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 5 Jan 2015 23:16:59 -0800 Subject: [PATCH 52/67] Adjust documents for distributed filer metadata store. --- docs/distributed_filer.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/distributed_filer.rst b/docs/distributed_filer.rst index 2783474815..29055c73e9 100644 --- a/docs/distributed_filer.rst +++ b/docs/distributed_filer.rst @@ -44,7 +44,7 @@ To start a weed filer in distributed mode: # assuming you already started weed master and weed volume weed filer -cassandra.server=localhost -Now you can add/delete files, and even browse the sub directories and files +Now you can add/delete files .. code-block:: bash @@ -73,6 +73,10 @@ A flat namespace would take more space because the parent directories are repeatedly stored. But disk space is a lesser concern especially for distributed systems. +The Cassandra table is a simple file_full_path ~ file_id mapping. Actually +it is a file_full_path ~ list_of_file_ids mapping with the hope to support +easy file appending for streaming files. + Complexity ################### @@ -83,14 +87,15 @@ take longer than the Cassandra internal lookup. Use Cases ######################### -Clients can assess one "weed filer" via HTTP, list files under a directory, create files via HTTP POST, read files via HTTP POST directly. - -Although one "weed filer" can only sits in one machine, you can start multiple "weed filer" on several machines, each "weed filer" instance running in its own collection, having its own namespace, but sharing the same Seaweed-FS storage. +Clients can assess one "weed filer" via HTTP, create files via HTTP POST, +read files via HTTP POST directly. Future ################### -The Cassandra implementation can be switched to other distributed hash table. +SeaweedFS can support additional distributed databases. It will be better +if that database can support prefix search, in order to list files +under a directory. Helps Wanted ######################## From 25568245cf3964df63b5982312363b517595d8c9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 5 Jan 2015 23:20:42 -0800 Subject: [PATCH 53/67] Adjust minor document text. --- docs/index.rst | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 429e76d64e..6692df94a4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,16 +3,11 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to weed-fs's documentation! +Welcome to Seaweed-FS documentation! =================================== -An official mirror of code.google.com/p/weed-fs . -Moving to github.com to make cooperations easier. -This repo and the google code repo will be kept synchronized. - - -For documents and bug reporting, Please visit - http://weed-fs.googlecode.com +This is the official site for Seaweed-FS. +The one on google code is deprecated. For pre-compiled releases, https://bintray.com/chrislusf/Weed-FS/seaweed From 7471bccb9bd0705725c838b867f3edde72ab082d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 5 Jan 2015 23:24:12 -0800 Subject: [PATCH 54/67] Adjust document script. --- docs/conf.py | 10 +++++----- docs/index.rst | 9 --------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index b70feab4e4..66fc9b3b4f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -43,8 +43,8 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'weed-fs' -copyright = u'2014, chrislusf, ernado' +project = u'seaweed-fs' +copyright = u'2015, chrislusf, ernado' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -240,8 +240,8 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'weed-fs', u'weed-fs Documentation', - u'chrislusf, ernado', 'weed-fs', 'One line description of project.', + ('index', 'seaweed-fs', u'seaweed-fs Documentation', + u'chrislusf, ernado', 'seaweed-fs', 'One line description of project.', 'Miscellaneous'), ] @@ -263,4 +263,4 @@ on_rtd = os.environ.get('READTHEDOCS', None) == 'True' if not on_rtd: # only import and set the theme if we're building docs locally import sphinx_rtd_theme html_theme = 'sphinx_rtd_theme' - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] \ No newline at end of file + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] diff --git a/docs/index.rst b/docs/index.rst index 6692df94a4..0775060070 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,12 +30,3 @@ Contents: optimization benchmarks changelist - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - From 0b192917f824e90ff7e9fc35080973f96c04fb45 Mon Sep 17 00:00:00 2001 From: yanyiwu Date: Tue, 6 Jan 2015 19:31:26 +0800 Subject: [PATCH 55/67] fix bug about the collection in volume is wrong when curl -F "file=@/tmp/test.jpg" "localhost:9333/submit?collection=picture" --- go/weed/weed_server/common.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index 5c782c091a..38d58bdbbc 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -87,6 +87,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st } debug("assigning file id for", fname) + r.ParseForm() assignResult, ae := operation.Assign(masterUrl, 1, r.FormValue("replication"), r.FormValue("collection"), r.FormValue("ttl")) if ae != nil { writeJsonError(w, r, ae) From cceb5768c799b36d7f2067281ad814d216060242 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 6 Jan 2015 19:26:48 -0800 Subject: [PATCH 56/67] Adjust function name. --- go/weed/filer.go | 2 +- go/weed/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/weed/filer.go b/go/weed/filer.go index 767864450d..ce4bdf4b90 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -67,7 +67,7 @@ func runFiler(cmd *Command, args []string) bool { } r := http.NewServeMux() - _, nfs_err := weed_server.NewEmbeddedFilerServer(r, *f.port, *f.master, *f.dir, *f.collection, + _, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection, *f.defaultReplicaPlacement, *f.redirectOnRead, *f.cassandra_server, *f.cassandra_keyspace, ) diff --git a/go/weed/server.go b/go/weed/server.go index 3aae2e7fb8..c5be89927a 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -157,7 +157,7 @@ func runServer(cmd *Command, args []string) bool { if *isStartingFiler { go func() { r := http.NewServeMux() - _, nfs_err := weed_server.NewEmbeddedFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection, + _, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection, *filerOptions.defaultReplicaPlacement, *filerOptions.redirectOnRead, "", "", ) From d77f3120c3b33162a6bf7971fcefc342b8a571de Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 6 Jan 2015 19:34:11 -0800 Subject: [PATCH 57/67] Adjust function name. --- go/weed/weed_server/filer_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index 325c9b6267..0ac1ab24b7 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -20,7 +20,7 @@ type FilerServer struct { filer filer.Filer } -func NewEmbeddedFilerServer(r *http.ServeMux, port int, master string, dir string, collection string, +func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string, replication string, redirectOnRead bool, cassandra_server string, cassandra_keyspace string, ) (fs *FilerServer, err error) { From e4531fc1e6003a2114b46ec3f6dacd4a65b6a9a5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 6 Jan 2015 20:15:13 -0800 Subject: [PATCH 58/67] Add Redis support for Distributed Filer store. --- docs/distributed_filer.rst | 31 +++++++++++++------ go/filer/redis_store/redis_store.go | 48 +++++++++++++++++++++++++++++ go/weed/filer.go | 5 +++ go/weed/server.go | 1 + go/weed/weed_server/filer_server.go | 19 +++++++----- 5 files changed, 87 insertions(+), 17 deletions(-) create mode 100644 go/filer/redis_store/redis_store.go diff --git a/docs/distributed_filer.rst b/docs/distributed_filer.rst index 29055c73e9..ad9f85d5c8 100644 --- a/docs/distributed_filer.rst +++ b/docs/distributed_filer.rst @@ -10,7 +10,11 @@ However, no SPOF is a must-have requirement for many projects. Luckily, SeaweedFS is so flexible that we can use a completely different way to manage file metadata. -This distributed filer uses Cassandra to store the metadata. +This distributed filer uses Redis or Cassandra to store the metadata. + +Redis Setup +##################### +No setup required. Cassandra Setup ##################### @@ -37,7 +41,14 @@ For production server, you would want to set replication_factor to 3. Sample usage ##################### -To start a weed filer in distributed mode: +To start a weed filer in distributed mode with Redis: + +.. code-block:: bash + + # assuming you already started weed master and weed volume + weed filer -redis.server=localhost:6379 + +To start a weed filer in distributed mode with Cassandra: .. code-block:: bash @@ -57,8 +68,8 @@ Now you can add/delete files Limitation ############ -List sub folders and files are not supported because Cassandra does not support -prefix search. +List sub folders and files are not supported because Redis or Cassandra +does not support prefix search. Flat Namespace Design ############ @@ -73,16 +84,16 @@ A flat namespace would take more space because the parent directories are repeatedly stored. But disk space is a lesser concern especially for distributed systems. -The Cassandra table is a simple file_full_path ~ file_id mapping. Actually -it is a file_full_path ~ list_of_file_ids mapping with the hope to support -easy file appending for streaming files. +So either Redis or Cassandra is a simple file_full_path ~ file_id mapping. +(Actually Cassandra is a file_full_path ~ list_of_file_ids mapping +with the hope to support easy file appending for streaming files.) Complexity ################### For one file retrieval, the full_filename=>file_id lookup will be O(logN) -using Cassandra. But very likely the one additional network hop would -take longer than the Cassandra internal lookup. +using Redis or Cassandra. But very likely the one additional network hop would +take longer than the actual lookup. Use Cases ######################### @@ -93,7 +104,7 @@ read files via HTTP POST directly. Future ################### -SeaweedFS can support additional distributed databases. It will be better +SeaweedFS can support other distributed databases. It will be better if that database can support prefix search, in order to list files under a directory. diff --git a/go/filer/redis_store/redis_store.go b/go/filer/redis_store/redis_store.go new file mode 100644 index 0000000000..8fe1c7ca59 --- /dev/null +++ b/go/filer/redis_store/redis_store.go @@ -0,0 +1,48 @@ +package redis_store + +import ( + redis "gopkg.in/redis.v2" +) + +type RedisStore struct { + Client *redis.Client +} + +func NewRedisStore(hostPort string, database int) *RedisStore { + client := redis.NewTCPClient(&redis.Options{ + Addr: hostPort, + Password: "", // no password set + DB: int64(database), + }) + return &RedisStore{Client: client} +} + +func (s *RedisStore) Get(fullFileName string) (fid string, err error) { + fid, err = s.Client.Get(fullFileName).Result() + if err == redis.Nil { + err = nil + } + return fid, err +} +func (s *RedisStore) Put(fullFileName string, fid string) (err error) { + _, err = s.Client.Set(fullFileName, fid).Result() + if err == redis.Nil { + err = nil + } + return err +} + +// Currently the fid is not returned +func (s *RedisStore) Delete(fullFileName string) (fid string, err error) { + _, err = s.Client.Del(fullFileName).Result() + if err == redis.Nil { + err = nil + } + return "", err +} + +func (c *RedisStore) Close() { + if c.Client != nil { + c.Client.Close() + } +} diff --git a/go/weed/filer.go b/go/weed/filer.go index ce4bdf4b90..bc528be33e 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -24,6 +24,8 @@ type FilerOptions struct { redirectOnRead *bool cassandra_server *string cassandra_keyspace *string + redis_server *string + redis_database *int } func init() { @@ -36,6 +38,8 @@ func init() { f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server") f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server") + f.redis_server = cmdFiler.Flag.String("redis.server", "", "host:port of the cassandra server, e.g., 127.0.0.1:6379") + f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server") } var cmdFiler = &Command{ @@ -70,6 +74,7 @@ func runFiler(cmd *Command, args []string) bool { _, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection, *f.defaultReplicaPlacement, *f.redirectOnRead, *f.cassandra_server, *f.cassandra_keyspace, + *f.redis_server, *f.redis_database, ) if nfs_err != nil { glog.Fatalf(nfs_err.Error()) diff --git a/go/weed/server.go b/go/weed/server.go index c5be89927a..062b02477a 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -160,6 +160,7 @@ func runServer(cmd *Command, args []string) bool { _, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection, *filerOptions.defaultReplicaPlacement, *filerOptions.redirectOnRead, "", "", + "", 0, ) if nfs_err != nil { glog.Fatalf(nfs_err.Error()) diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go index 0ac1ab24b7..18a02b5e0f 100644 --- a/go/weed/weed_server/filer_server.go +++ b/go/weed/weed_server/filer_server.go @@ -8,6 +8,7 @@ import ( "github.com/chrislusf/weed-fs/go/filer/cassandra_store" "github.com/chrislusf/weed-fs/go/filer/embedded_filer" "github.com/chrislusf/weed-fs/go/filer/flat_namespace" + "github.com/chrislusf/weed-fs/go/filer/redis_store" "github.com/chrislusf/weed-fs/go/glog" ) @@ -23,6 +24,7 @@ type FilerServer struct { func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string, replication string, redirectOnRead bool, cassandra_server string, cassandra_keyspace string, + redis_server string, redis_database int, ) (fs *FilerServer, err error) { fs = &FilerServer{ master: master, @@ -32,19 +34,22 @@ func NewFilerServer(r *http.ServeMux, port int, master string, dir string, colle port: ":" + strconv.Itoa(port), } - if cassandra_server == "" { + if cassandra_server != "" { + cassandra_store, err := cassandra_store.NewCassandraStore(cassandra_keyspace, cassandra_server) + if err != nil { + glog.Fatalf("Can not connect to cassandra server %s with keyspace %s: %v", cassandra_server, cassandra_keyspace, err) + } + fs.filer = flat_namespace.NewFlatNamesapceFiler(master, cassandra_store) + } else if redis_server != "" { + redis_store := redis_store.NewRedisStore(redis_server, redis_database) + fs.filer = flat_namespace.NewFlatNamesapceFiler(master, redis_store) + } else { if fs.filer, err = embedded_filer.NewFilerEmbedded(master, dir); err != nil { glog.Fatalf("Can not start filer in dir %s : %v", err) return } r.HandleFunc("/admin/mv", fs.moveHandler) - } else { - cassandra_store, err := cassandra_store.NewCassandraStore(cassandra_keyspace, cassandra_server) - if err != nil { - glog.Fatalf("Can not connect to cassandra server %s with keyspace %s: %v", cassandra_server, cassandra_keyspace, err) - } - fs.filer = flat_namespace.NewFlatNamesapceFiler(master, cassandra_store) } r.HandleFunc("/", fs.filerHandler) From f7f6f331dbadbdd43ac39e5dc2b1604a58c4076b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 6 Jan 2015 22:38:56 -0800 Subject: [PATCH 59/67] Adjust documentation. --- docs/distributed_filer.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/distributed_filer.rst b/docs/distributed_filer.rst index ad9f85d5c8..b4bd9a43af 100644 --- a/docs/distributed_filer.rst +++ b/docs/distributed_filer.rst @@ -20,7 +20,8 @@ Cassandra Setup ##################### Here is the CQL to create the table.CassandraStore. Optionally you can adjust the keyspace name and replication settings. -For production server, you would want to set replication_factor to 3. +For production, you would want to set replication_factor to 3 +if there are at least 3 Cassandra servers. .. code-block:: bash From 7d810a15c01587a74708a56b5b9739e7dbb0858a Mon Sep 17 00:00:00 2001 From: yanyiwu Date: Wed, 7 Jan 2015 16:47:57 +0800 Subject: [PATCH 60/67] fix bug about : writed needle into localfile twice when replicating, because of the ip is not equal : ":8080" != "127.0.0.1:8080" --- go/weed/volume.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/weed/volume.go b/go/weed/volume.go index 22b50cd346..1683e19272 100644 --- a/go/weed/volume.go +++ b/go/weed/volume.go @@ -71,6 +71,7 @@ func runVolume(cmd *Command, args []string) bool { if *publicIp == "" { if *ip == "" { + *ip = "127.0.0.1" *publicIp = "localhost" } else { *publicIp = *ip From cacfc8586957220423210a496e3064d7eafab393 Mon Sep 17 00:00:00 2001 From: yanyiwu Date: Thu, 8 Jan 2015 15:54:50 +0800 Subject: [PATCH 61/67] add some String() to make codes easier to read and debug --- go/operation/lookup.go | 6 +++++- go/storage/volume_info.go | 5 +++++ go/topology/collection.go | 6 ++++++ go/topology/data_node.go | 5 +++++ go/topology/volume_growth.go | 4 ++++ go/topology/volume_layout.go | 5 +++++ go/topology/volume_location_list.go | 8 ++++++++ go/util/concurrent_read_map.go | 4 +++- 8 files changed, 41 insertions(+), 2 deletions(-) diff --git a/go/operation/lookup.go b/go/operation/lookup.go index e6b6658daa..70bc7146ec 100644 --- a/go/operation/lookup.go +++ b/go/operation/lookup.go @@ -3,7 +3,7 @@ package operation import ( "encoding/json" "errors" - _ "fmt" + "fmt" "math/rand" "net/url" "strings" @@ -22,6 +22,10 @@ type LookupResult struct { Error string `json:"error,omitempty"` } +func (lr *LookupResult) String() string { + return fmt.Sprintf("VolumeId:%s, Locations:%v, Error:%s", lr.VolumeId, lr.Locations, lr.Error) +} + var ( vc VidCache ) diff --git a/go/storage/volume_info.go b/go/storage/volume_info.go index 6410c17840..bc8049ea44 100644 --- a/go/storage/volume_info.go +++ b/go/storage/volume_info.go @@ -1,6 +1,7 @@ package storage import ( + "fmt" "github.com/chrislusf/weed-fs/go/operation" ) @@ -36,3 +37,7 @@ func NewVolumeInfo(m *operation.VolumeInformationMessage) (vi VolumeInfo, err er vi.Ttl = LoadTTLFromUint32(*m.Ttl) return vi, nil } + +func (vi VolumeInfo) String() string { + return fmt.Sprintf("Id:%s, Size:%d, ReplicaPlacement:%s, Collection:%s, Version:%v, FileCount:%d, DeleteCount:%d, DeletedByteCount:%d, ReadOnly:%v", vi.Id, vi.Size, vi.ReplicaPlacement, vi.Collection, vi.Version, vi.FileCount, vi.DeleteCount, vi.DeletedByteCount, vi.ReadOnly) +} diff --git a/go/topology/collection.go b/go/topology/collection.go index 4b47ae88a1..5437ffd79d 100644 --- a/go/topology/collection.go +++ b/go/topology/collection.go @@ -1,6 +1,8 @@ package topology import ( + "fmt" + "github.com/chrislusf/weed-fs/go/storage" "github.com/chrislusf/weed-fs/go/util" ) @@ -17,6 +19,10 @@ func NewCollection(name string, volumeSizeLimit uint64) *Collection { return c } +func (c *Collection) String() string { + return fmt.Sprintf("Name:%s, volumeSizeLimit:%d, storageType2VolumeLayout:%v", c.Name, c.volumeSizeLimit, c.storageType2VolumeLayout) +} + func (c *Collection) GetOrCreateVolumeLayout(rp *storage.ReplicaPlacement, ttl *storage.TTL) *VolumeLayout { keyString := rp.String() if ttl != nil { diff --git a/go/topology/data_node.go b/go/topology/data_node.go index 109bd037fe..09b9fac6c4 100644 --- a/go/topology/data_node.go +++ b/go/topology/data_node.go @@ -1,6 +1,7 @@ package topology import ( + "fmt" "strconv" "github.com/chrislusf/weed-fs/go/glog" @@ -26,6 +27,10 @@ func NewDataNode(id string) *DataNode { return s } +func (dn *DataNode) String() string { + return fmt.Sprintf("NodeImpl:%s ,volumes:%v, Ip:%s, Port:%d, PublicUrl:%s, Dead:%v", dn.NodeImpl.String(), dn.volumes, dn.Ip, dn.Port, dn.PublicUrl, dn.Dead) +} + func (dn *DataNode) AddOrUpdateVolume(v storage.VolumeInfo) { if _, ok := dn.volumes[v.Id]; !ok { dn.volumes[v.Id] = v diff --git a/go/topology/volume_growth.go b/go/topology/volume_growth.go index b1f241990b..6124c0da21 100644 --- a/go/topology/volume_growth.go +++ b/go/topology/volume_growth.go @@ -30,6 +30,10 @@ type VolumeGrowth struct { accessLock sync.Mutex } +func (o *VolumeGrowOption) String() string { + return fmt.Sprintf("Collection:%s, ReplicaPlacement:%v, Ttl:%v, DataCenter:%s, Rack:%s, DataNode:%s", o.Collection, o.ReplicaPlacement, o.Ttl, o.DataCenter, o.Rack, o.DataNode) +} + func NewDefaultVolumeGrowth() *VolumeGrowth { return &VolumeGrowth{} } diff --git a/go/topology/volume_layout.go b/go/topology/volume_layout.go index de72bf8952..4b1d3dad96 100644 --- a/go/topology/volume_layout.go +++ b/go/topology/volume_layout.go @@ -2,6 +2,7 @@ package topology import ( "errors" + "fmt" "math/rand" "sync" @@ -29,6 +30,10 @@ func NewVolumeLayout(rp *storage.ReplicaPlacement, ttl *storage.TTL, volumeSizeL } } +func (vl *VolumeLayout) String() string { + return fmt.Sprintf("rp:%v, ttl:%v, vid2location:%v, writables:%v, volumeSizeLimit:%v", vl.rp, vl.ttl, vl.vid2location, vl.writables, vl.volumeSizeLimit) +} + func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) { vl.accessLock.Lock() defer vl.accessLock.Unlock() diff --git a/go/topology/volume_location_list.go b/go/topology/volume_location_list.go index 0116140137..0f892c010f 100644 --- a/go/topology/volume_location_list.go +++ b/go/topology/volume_location_list.go @@ -1,5 +1,9 @@ package topology +import ( + "fmt" +) + type VolumeLocationList struct { list []*DataNode } @@ -8,6 +12,10 @@ func NewVolumeLocationList() *VolumeLocationList { return &VolumeLocationList{} } +func (dnll *VolumeLocationList) String() string { + return fmt.Sprintf("%v", dnll.list) +} + func (dnll *VolumeLocationList) Head() *DataNode { return dnll.list[0] } diff --git a/go/util/concurrent_read_map.go b/go/util/concurrent_read_map.go index d16fdbcaf3..880fe54e37 100644 --- a/go/util/concurrent_read_map.go +++ b/go/util/concurrent_read_map.go @@ -1,6 +1,8 @@ package util -import "sync" +import ( + "sync" +) // A mostly for read map, which can thread-safely // initialize the map entries. From 5b5d6341e570b2f12cd375279ee004529193b970 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 8 Jan 2015 00:19:32 -0800 Subject: [PATCH 62/67] Set http status after modifying response headers. --- go/weed/weed_server/common.go | 33 ++++++++++--------- go/weed/weed_server/filer_server_handlers.go | 24 +++++++------- .../filer_server_handlers_admin.go | 2 +- go/weed/weed_server/master_server.go | 4 ++- go/weed/weed_server/master_server_handlers.go | 22 ++++++------- .../master_server_handlers_admin.go | 29 ++++++++-------- go/weed/weed_server/raft_server_handlers.go | 2 +- go/weed/weed_server/volume_server_handlers.go | 31 ++++++++--------- .../volume_server_handlers_admin.go | 16 ++++----- .../volume_server_handlers_vacuum.go | 12 +++---- 10 files changed, 87 insertions(+), 88 deletions(-) diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index 38d58bdbbc..3968f3191b 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -3,6 +3,7 @@ package weed_server import ( "bytes" "encoding/json" + "errors" "fmt" "net/http" "path/filepath" @@ -24,7 +25,7 @@ func init() { } -func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) (err error) { +func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) (err error) { var bytes []byte if r.FormValue("pretty") != "" { bytes, err = json.MarshalIndent(obj, "", " ") @@ -37,9 +38,11 @@ func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) (err err callback := r.FormValue("callback") if callback == "" { w.Header().Set("Content-Type", "application/json") + w.WriteHeader(httpStatus) _, err = w.Write(bytes) } else { w.Header().Set("Content-Type", "application/javascript") + w.WriteHeader(httpStatus) if _, err = w.Write([]uint8(callback)); err != nil { return } @@ -51,20 +54,21 @@ func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) (err err return } } + return } // wrapper for writeJson - just logs errors -func writeJsonQuiet(w http.ResponseWriter, r *http.Request, obj interface{}) { - if err := writeJson(w, r, obj); err != nil { +func writeJsonQuiet(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) { + if err := writeJson(w, r, httpStatus, obj); err != nil { glog.V(0).Infof("error writing JSON %s: %s", obj, err.Error()) } } -func writeJsonError(w http.ResponseWriter, r *http.Request, err error) { +func writeJsonError(w http.ResponseWriter, r *http.Request, httpStatus int, err error) { w.WriteHeader(http.StatusInternalServerError) m := make(map[string]interface{}) m["error"] = err.Error() - writeJsonQuiet(w, r, m) + writeJsonQuiet(w, r, httpStatus, m) } func debug(params ...interface{}) { @@ -74,15 +78,14 @@ func debug(params ...interface{}) { func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl string) { m := make(map[string]interface{}) if r.Method != "POST" { - m["error"] = "Only submit via POST!" - writeJsonQuiet(w, r, m) + writeJsonError(w, r, http.StatusMethodNotAllowed, errors.New("Only submit via POST!")) return } debug("parsing upload file...") fname, data, mimeType, isGzipped, lastModified, _, pe := storage.ParseUpload(r) if pe != nil { - writeJsonError(w, r, pe) + writeJsonError(w, r, http.StatusBadRequest, pe) return } @@ -90,7 +93,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st r.ParseForm() assignResult, ae := operation.Assign(masterUrl, 1, r.FormValue("replication"), r.FormValue("collection"), r.FormValue("ttl")) if ae != nil { - writeJsonError(w, r, ae) + writeJsonError(w, r, http.StatusInternalServerError, ae) return } @@ -102,7 +105,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st debug("upload file to store", url) uploadResult, err := operation.Upload(url, fname, bytes.NewReader(data), isGzipped, mimeType) if err != nil { - writeJsonError(w, r, err) + writeJsonError(w, r, http.StatusInternalServerError, err) return } @@ -110,7 +113,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st m["fid"] = assignResult.Fid m["fileUrl"] = assignResult.PublicUrl + "/" + assignResult.Fid m["size"] = uploadResult.Size - writeJsonQuiet(w, r, m) + writeJsonQuiet(w, r, http.StatusCreated, m) return } @@ -119,10 +122,10 @@ func deleteForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st fids := r.Form["fid"] ret, err := operation.DeleteFiles(masterUrl, fids) if err != nil { - writeJsonError(w, r, err) + writeJsonError(w, r, http.StatusInternalServerError, err) return } - writeJsonQuiet(w, r, ret) + writeJsonQuiet(w, r, http.StatusAccepted, ret) } func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly bool) { @@ -162,12 +165,12 @@ func statsCounterHandler(w http.ResponseWriter, r *http.Request) { m := make(map[string]interface{}) m["Version"] = util.VERSION m["Counters"] = serverStats - writeJsonQuiet(w, r, m) + writeJsonQuiet(w, r, http.StatusOK, m) } func statsMemoryHandler(w http.ResponseWriter, r *http.Request) { m := make(map[string]interface{}) m["Version"] = util.VERSION m["Memory"] = stats.MemStat() - writeJsonQuiet(w, r, m) + writeJsonQuiet(w, r, http.StatusOK, m) } diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index 1afc1f8521..29a3d4b079 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -50,7 +50,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque limit = 100 } m["Files"], _ = fs.filer.ListFiles(r.URL.Path, lastFileName, limit) - writeJsonQuiet(w, r, m) + writeJsonQuiet(w, r, http.StatusOK, m) } func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) { if strings.HasSuffix(r.URL.Path, "/") { @@ -102,7 +102,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, resp, do_err := util.Do(request) if do_err != nil { glog.V(0).Infoln("failing to connect to volume server", do_err.Error()) - writeJsonError(w, r, do_err) + writeJsonError(w, r, http.StatusInternalServerError, do_err) return } defer resp.Body.Close() @@ -122,7 +122,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { assignResult, ae := operation.Assign(fs.master, 1, replication, fs.collection, query.Get("ttl")) if ae != nil { glog.V(0).Infoln("failing to assign a file id", ae.Error()) - writeJsonError(w, r, ae) + writeJsonError(w, r, http.StatusInternalServerError, ae) return } @@ -142,14 +142,14 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { resp, do_err := util.Do(request) if do_err != nil { glog.V(0).Infoln("failing to connect to volume server", r.RequestURI, do_err.Error()) - writeJsonError(w, r, do_err) + writeJsonError(w, r, http.StatusInternalServerError, do_err) return } defer resp.Body.Close() resp_body, ra_err := ioutil.ReadAll(resp.Body) if ra_err != nil { glog.V(0).Infoln("failing to upload to volume server", r.RequestURI, ra_err.Error()) - writeJsonError(w, r, ra_err) + writeJsonError(w, r, http.StatusInternalServerError, ra_err) return } glog.V(4).Infoln("post result", string(resp_body)) @@ -157,12 +157,12 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { unmarshal_err := json.Unmarshal(resp_body, &ret) if unmarshal_err != nil { glog.V(0).Infoln("failing to read upload resonse", r.RequestURI, string(resp_body)) - writeJsonError(w, r, unmarshal_err) + writeJsonError(w, r, http.StatusInternalServerError, unmarshal_err) return } if ret.Error != "" { glog.V(0).Infoln("failing to post to volume server", r.RequestURI, ret.Error) - writeJsonError(w, r, errors.New(ret.Error)) + writeJsonError(w, r, http.StatusInternalServerError, errors.New(ret.Error)) return } path := r.URL.Path @@ -172,7 +172,8 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { } else { operation.DeleteFile(fs.master, assignResult.Fid) //clean up glog.V(0).Infoln("Can not to write to folder", path, "without a file name!") - writeJsonError(w, r, errors.New("Can not to write to folder "+path+" without a file name")) + writeJsonError(w, r, http.StatusInternalServerError, + errors.New("Can not to write to folder "+path+" without a file name")) return } } @@ -180,7 +181,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { if db_err := fs.filer.CreateFile(path, assignResult.Fid); db_err != nil { operation.DeleteFile(fs.master, assignResult.Fid) //clean up glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err) - writeJsonError(w, r, db_err) + writeJsonError(w, r, http.StatusInternalServerError, db_err) return } w.WriteHeader(http.StatusCreated) @@ -201,10 +202,9 @@ func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { } } if err == nil { - w.WriteHeader(http.StatusAccepted) - writeJsonQuiet(w, r, map[string]string{"error": ""}) + writeJsonQuiet(w, r, http.StatusAccepted, map[string]string{"error": ""}) } else { glog.V(4).Infoln("deleting", r.URL.Path, ":", err.Error()) - writeJsonError(w, r, err) + writeJsonError(w, r, http.StatusInternalServerError, err) } } diff --git a/go/weed/weed_server/filer_server_handlers_admin.go b/go/weed/weed_server/filer_server_handlers_admin.go index 5ba12e0b8f..3f67b30f15 100644 --- a/go/weed/weed_server/filer_server_handlers_admin.go +++ b/go/weed/weed_server/filer_server_handlers_admin.go @@ -22,7 +22,7 @@ func (fs *FilerServer) moveHandler(w http.ResponseWriter, r *http.Request) { err := fs.filer.Move(from, to) if err != nil { glog.V(4).Infoln("moving", from, "->", to, err.Error()) - writeJsonError(w, r, err) + writeJsonError(w, r, http.StatusInternalServerError, err) } else { w.WriteHeader(http.StatusOK) } diff --git a/go/weed/weed_server/master_server.go b/go/weed/weed_server/master_server.go index 0a4ca5d5ea..056b1fe7b4 100644 --- a/go/weed/weed_server/master_server.go +++ b/go/weed/weed_server/master_server.go @@ -1,6 +1,7 @@ package weed_server import ( + "fmt" "net/http" "net/http/httputil" "net/url" @@ -103,7 +104,8 @@ func (ms *MasterServer) proxyToLeader(f func(w http.ResponseWriter, r *http.Requ defer func() { <-ms.bounedLeaderChan }() targetUrl, err := url.Parse("http://" + ms.Topo.RaftServer.Leader()) if err != nil { - writeJsonQuiet(w, r, map[string]interface{}{"error": "Leader URL http://" + ms.Topo.RaftServer.Leader() + " Parse Error " + err.Error()}) + writeJsonError(w, r, http.StatusInternalServerError, + fmt.Errorf("Leader URL http://%s Parse Error: %v", ms.Topo.RaftServer.Leader(), err)) return } glog.V(4).Infoln("proxying to leader", ms.Topo.RaftServer.Leader()) diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go index 7a7a3b70d9..7b6ce58f32 100644 --- a/go/weed/weed_server/master_server_handlers.go +++ b/go/weed/weed_server/master_server_handlers.go @@ -1,6 +1,7 @@ package weed_server import ( + "fmt" "net/http" "strconv" "strings" @@ -50,10 +51,11 @@ func (ms *MasterServer) dirLookupHandler(w http.ResponseWriter, r *http.Request) collection := r.FormValue("collection") //optional, but can be faster if too many collections volumeLocations := ms.lookupVolumeId(vids, collection) location := volumeLocations[vid] + httpStatus := http.StatusOK if location.Error != "" { - w.WriteHeader(http.StatusNotFound) + httpStatus = http.StatusNotFound } - writeJsonQuiet(w, r, location) + writeJsonQuiet(w, r, httpStatus, location) } // This can take batched volumeIds, &volumeId=x&volumeId=y&volumeId=z @@ -62,7 +64,7 @@ func (ms *MasterServer) volumeLookupHandler(w http.ResponseWriter, r *http.Reque vids := r.Form["volumeId"] collection := r.FormValue("collection") //optional, but can be faster if too many collections volumeLocations := ms.lookupVolumeId(vids, collection) - writeJsonQuiet(w, r, volumeLocations) + writeJsonQuiet(w, r, http.StatusOK, volumeLocations) } func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) { @@ -74,22 +76,21 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) option, err := ms.getVolumeGrowOption(r) if err != nil { - w.WriteHeader(http.StatusNotAcceptable) - writeJsonQuiet(w, r, operation.AssignResult{Error: err.Error()}) + writeJsonQuiet(w, r, http.StatusNotAcceptable, operation.AssignResult{Error: err.Error()}) return } if !ms.Topo.HasWritableVolume(option) { if ms.Topo.FreeSpace() <= 0 { - w.WriteHeader(http.StatusNotFound) - writeJsonQuiet(w, r, operation.AssignResult{Error: "No free volumes left!"}) + writeJsonQuiet(w, r, http.StatusNotFound, operation.AssignResult{Error: "No free volumes left!"}) return } else { ms.vgLock.Lock() defer ms.vgLock.Unlock() if !ms.Topo.HasWritableVolume(option) { if _, err = ms.vg.AutomaticGrowByType(option, ms.Topo); err != nil { - writeJsonQuiet(w, r, operation.AssignResult{Error: "Cannot grow volume group! " + err.Error()}) + writeJsonError(w, r, http.StatusInternalServerError, + fmt.Errorf("Cannot grow volume group! %v", err)) return } } @@ -97,9 +98,8 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) } fid, count, dn, err := ms.Topo.PickForWrite(requestedCount, option) if err == nil { - writeJsonQuiet(w, r, operation.AssignResult{Fid: fid, Url: dn.Url(), PublicUrl: dn.PublicUrl, Count: count}) + writeJsonQuiet(w, r, http.StatusOK, operation.AssignResult{Fid: fid, Url: dn.Url(), PublicUrl: dn.PublicUrl, Count: count}) } else { - w.WriteHeader(http.StatusNotAcceptable) - writeJsonQuiet(w, r, operation.AssignResult{Error: err.Error()}) + writeJsonQuiet(w, r, http.StatusNotAcceptable, operation.AssignResult{Error: err.Error()}) } } diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go index 75e0593658..437044eee1 100644 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ b/go/weed/weed_server/master_server_handlers_admin.go @@ -3,6 +3,7 @@ package weed_server import ( "encoding/json" "errors" + "fmt" "io/ioutil" "net/http" "strconv" @@ -19,13 +20,13 @@ import ( func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) { collection, ok := ms.Topo.GetCollection(r.FormValue("collection")) if !ok { - writeJsonQuiet(w, r, map[string]interface{}{"error": "collection " + r.FormValue("collection") + "does not exist!"}) + writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("collection %s does not exist!", r.FormValue("collection"))) return } for _, server := range collection.ListVolumeServers() { _, err := util.Get("http://" + server.Ip + ":" + strconv.Itoa(server.Port) + "/admin/delete_collection?collection=" + r.FormValue("collection")) if err != nil { - writeJsonQuiet(w, r, map[string]string{"error": err.Error()}) + writeJsonError(w, r, http.StatusInternalServerError, err) return } } @@ -35,12 +36,12 @@ func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.R func (ms *MasterServer) dirJoinHandler(w http.ResponseWriter, r *http.Request) { body, err := ioutil.ReadAll(r.Body) if err != nil { - writeJsonError(w, r, err) + writeJsonError(w, r, http.StatusBadRequest, err) return } joinMessage := &operation.JoinMessage{} if err = proto.Unmarshal(body, joinMessage); err != nil { - writeJsonError(w, r, err) + writeJsonError(w, r, http.StatusBadRequest, err) return } if *joinMessage.Ip == "" { @@ -49,7 +50,7 @@ func (ms *MasterServer) dirJoinHandler(w http.ResponseWriter, r *http.Request) { if glog.V(4) { if jsonData, jsonError := json.Marshal(joinMessage); jsonError != nil { glog.V(0).Infoln("json marshaling error: ", jsonError) - writeJsonError(w, r, jsonError) + writeJsonError(w, r, http.StatusBadRequest, jsonError) return } else { glog.V(4).Infoln("Proto size", len(body), "json size", len(jsonData), string(jsonData)) @@ -57,14 +58,14 @@ func (ms *MasterServer) dirJoinHandler(w http.ResponseWriter, r *http.Request) { } ms.Topo.ProcessJoinMessage(joinMessage) - writeJsonQuiet(w, r, operation.JoinResult{VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024}) + writeJsonQuiet(w, r, http.StatusOK, operation.JoinResult{VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024}) } func (ms *MasterServer) dirStatusHandler(w http.ResponseWriter, r *http.Request) { m := make(map[string]interface{}) m["Version"] = util.VERSION m["Topology"] = ms.Topo.ToMap() - writeJsonQuiet(w, r, m) + writeJsonQuiet(w, r, http.StatusOK, m) } func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Request) { @@ -81,8 +82,7 @@ func (ms *MasterServer) volumeGrowHandler(w http.ResponseWriter, r *http.Request count := 0 option, err := ms.getVolumeGrowOption(r) if err != nil { - w.WriteHeader(http.StatusNotAcceptable) - writeJsonQuiet(w, r, map[string]string{"error": err.Error()}) + writeJsonError(w, r, http.StatusNotAcceptable, err) return } if err == nil { @@ -97,11 +97,9 @@ func (ms *MasterServer) volumeGrowHandler(w http.ResponseWriter, r *http.Request } } if err != nil { - w.WriteHeader(http.StatusNotAcceptable) - writeJsonQuiet(w, r, map[string]string{"error": err.Error()}) + writeJsonError(w, r, http.StatusNotAcceptable, err) } else { - w.WriteHeader(http.StatusOK) - writeJsonQuiet(w, r, map[string]interface{}{"count": count}) + writeJsonQuiet(w, r, http.StatusOK, map[string]interface{}{"count": count}) } } @@ -109,7 +107,7 @@ func (ms *MasterServer) volumeStatusHandler(w http.ResponseWriter, r *http.Reque m := make(map[string]interface{}) m["Version"] = util.VERSION m["Volumes"] = ms.Topo.ToVolumeMap() - writeJsonQuiet(w, r, m) + writeJsonQuiet(w, r, http.StatusOK, m) } func (ms *MasterServer) redirectHandler(w http.ResponseWriter, r *http.Request) { @@ -123,8 +121,7 @@ func (ms *MasterServer) redirectHandler(w http.ResponseWriter, r *http.Request) if machines != nil && len(machines) > 0 { http.Redirect(w, r, "http://"+machines[0].PublicUrl+r.URL.Path, http.StatusMovedPermanently) } else { - w.WriteHeader(http.StatusNotFound) - writeJsonQuiet(w, r, map[string]string{"error": "volume id " + volumeId.String() + " not found. "}) + writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("volume id %s not found.", volumeId)) } } diff --git a/go/weed/weed_server/raft_server_handlers.go b/go/weed/weed_server/raft_server_handlers.go index b466d9afac..629de248bd 100644 --- a/go/weed/weed_server/raft_server_handlers.go +++ b/go/weed/weed_server/raft_server_handlers.go @@ -60,5 +60,5 @@ func (s *RaftServer) statusHandler(w http.ResponseWriter, r *http.Request) { if leader, e := s.topo.Leader(); e == nil { ret.Leader = leader } - writeJsonQuiet(w, r, ret) + writeJsonQuiet(w, r, http.StatusOK, ret) } diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go index 3abfe96b58..6b47ee84da 100644 --- a/go/weed/weed_server/volume_server_handlers.go +++ b/go/weed/weed_server/volume_server_handlers.go @@ -1,6 +1,7 @@ package weed_server import ( + "errors" "io" "mime" "mime/multipart" @@ -235,35 +236,34 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) { if e := r.ParseForm(); e != nil { glog.V(0).Infoln("form parse error:", e) - writeJsonError(w, r, e) + writeJsonError(w, r, http.StatusBadRequest, e) return } vid, _, _, _, _ := parseURLPath(r.URL.Path) volumeId, ve := storage.NewVolumeId(vid) if ve != nil { glog.V(0).Infoln("NewVolumeId error:", ve) - writeJsonError(w, r, ve) + writeJsonError(w, r, http.StatusBadRequest, ve) return } needle, ne := storage.NewNeedle(r, vs.FixJpgOrientation) if ne != nil { - writeJsonError(w, r, ne) + writeJsonError(w, r, http.StatusBadRequest, ne) return } ret := operation.UploadResult{} size, errorStatus := topology.ReplicatedWrite(vs.masterNode, vs.store, volumeId, needle, r) - if errorStatus == "" { - w.WriteHeader(http.StatusCreated) - } else { - w.WriteHeader(http.StatusInternalServerError) + httpStatus := http.StatusCreated + if errorStatus != "" { + httpStatus = http.StatusInternalServerError ret.Error = errorStatus } if needle.HasName() { ret.Name = string(needle.Name) } ret.Size = size - writeJsonQuiet(w, r, ret) + writeJsonQuiet(w, r, httpStatus, ret) } func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { @@ -280,7 +280,7 @@ func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { if ok != nil { m := make(map[string]uint32) m["size"] = 0 - writeJsonQuiet(w, r, m) + writeJsonQuiet(w, r, http.StatusNotFound, m) return } @@ -293,14 +293,13 @@ func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { ret := topology.ReplicatedDelete(vs.masterNode, vs.store, volumeId, n, r) if ret != 0 { - w.WriteHeader(http.StatusAccepted) + m := make(map[string]uint32) + m["size"] = uint32(count) + writeJsonQuiet(w, r, http.StatusAccepted, m) } else { - w.WriteHeader(http.StatusInternalServerError) + writeJsonError(w, r, http.StatusInternalServerError, errors.New("Deletion Failed.")) } - m := make(map[string]uint32) - m["size"] = uint32(count) - writeJsonQuiet(w, r, m) } //Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas. @@ -334,7 +333,5 @@ func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Reques } } - w.WriteHeader(http.StatusAccepted) - - writeJsonQuiet(w, r, ret) + writeJsonQuiet(w, r, http.StatusAccepted, ret) } diff --git a/go/weed/weed_server/volume_server_handlers_admin.go b/go/weed/weed_server/volume_server_handlers_admin.go index caf4c3be8f..1581a5770f 100644 --- a/go/weed/weed_server/volume_server_handlers_admin.go +++ b/go/weed/weed_server/volume_server_handlers_admin.go @@ -13,15 +13,15 @@ func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { m := make(map[string]interface{}) m["Version"] = util.VERSION m["Volumes"] = vs.store.Status() - writeJsonQuiet(w, r, m) + writeJsonQuiet(w, r, http.StatusOK, m) } func (vs *VolumeServer) assignVolumeHandler(w http.ResponseWriter, r *http.Request) { err := vs.store.AddVolume(r.FormValue("volume"), r.FormValue("collection"), r.FormValue("replication"), r.FormValue("ttl")) if err == nil { - writeJsonQuiet(w, r, map[string]string{"error": ""}) + writeJsonQuiet(w, r, http.StatusAccepted, map[string]string{"error": ""}) } else { - writeJsonQuiet(w, r, map[string]string{"error": err.Error()}) + writeJsonError(w, r, http.StatusNotAcceptable, err) } glog.V(2).Infoln("assign volume =", r.FormValue("volume"), ", collection =", r.FormValue("collection"), ", replication =", r.FormValue("replication"), ", error =", err) } @@ -33,9 +33,9 @@ func (vs *VolumeServer) deleteCollectionHandler(w http.ResponseWriter, r *http.R } err := vs.store.DeleteCollection(r.FormValue("collection")) if err == nil { - writeJsonQuiet(w, r, map[string]string{"error": ""}) + writeJsonQuiet(w, r, http.StatusOK, map[string]string{"error": ""}) } else { - writeJsonQuiet(w, r, map[string]string{"error": err.Error()}) + writeJsonError(w, r, http.StatusInternalServerError, err) } glog.V(2).Infoln("deleting collection =", r.FormValue("collection"), ", error =", err) } @@ -44,9 +44,9 @@ func (vs *VolumeServer) freezeVolumeHandler(w http.ResponseWriter, r *http.Reque //TODO: notify master that this volume will be read-only err := vs.store.FreezeVolume(r.FormValue("volume")) if err == nil { - writeJsonQuiet(w, r, map[string]interface{}{"error": ""}) + writeJsonQuiet(w, r, http.StatusOK, map[string]string{"error": ""}) } else { - writeJsonQuiet(w, r, map[string]string{"error": err.Error()}) + writeJsonError(w, r, http.StatusInternalServerError, err) } glog.V(2).Infoln("freeze volume =", r.FormValue("volume"), ", error =", err) } @@ -61,5 +61,5 @@ func (vs *VolumeServer) statsDiskHandler(w http.ResponseWriter, r *http.Request) } } m["DiskStatues"] = ds - writeJsonQuiet(w, r, m) + writeJsonQuiet(w, r, http.StatusOK, m) } diff --git a/go/weed/weed_server/volume_server_handlers_vacuum.go b/go/weed/weed_server/volume_server_handlers_vacuum.go index f115e3b8b1..cb30e10b48 100644 --- a/go/weed/weed_server/volume_server_handlers_vacuum.go +++ b/go/weed/weed_server/volume_server_handlers_vacuum.go @@ -9,27 +9,27 @@ import ( func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) { err, ret := vs.store.CheckCompactVolume(r.FormValue("volume"), r.FormValue("garbageThreshold")) if err == nil { - writeJsonQuiet(w, r, map[string]interface{}{"error": "", "result": ret}) + writeJsonQuiet(w, r, http.StatusOK, map[string]interface{}{"error": "", "result": ret}) } else { - writeJsonQuiet(w, r, map[string]interface{}{"error": err.Error(), "result": false}) + writeJsonQuiet(w, r, http.StatusInternalServerError, map[string]interface{}{"error": err.Error(), "result": false}) } glog.V(2).Infoln("checked compacting volume =", r.FormValue("volume"), "garbageThreshold =", r.FormValue("garbageThreshold"), "vacuum =", ret) } func (vs *VolumeServer) vacuumVolumeCompactHandler(w http.ResponseWriter, r *http.Request) { err := vs.store.CompactVolume(r.FormValue("volume")) if err == nil { - writeJsonQuiet(w, r, map[string]string{"error": ""}) + writeJsonQuiet(w, r, http.StatusOK, map[string]string{"error": ""}) } else { - writeJsonQuiet(w, r, map[string]string{"error": err.Error()}) + writeJsonError(w, r, http.StatusInternalServerError, err) } glog.V(2).Infoln("compacted volume =", r.FormValue("volume"), ", error =", err) } func (vs *VolumeServer) vacuumVolumeCommitHandler(w http.ResponseWriter, r *http.Request) { err := vs.store.CommitCompactVolume(r.FormValue("volume")) if err == nil { - writeJsonQuiet(w, r, map[string]interface{}{"error": ""}) + writeJsonQuiet(w, r, http.StatusOK, map[string]string{"error": ""}) } else { - writeJsonQuiet(w, r, map[string]string{"error": err.Error()}) + writeJsonError(w, r, http.StatusInternalServerError, err) } glog.V(2).Infoln("commit compact volume =", r.FormValue("volume"), ", error =", err) } From cec2b3724c99cc86651c174ff3fba72280e0041f Mon Sep 17 00:00:00 2001 From: Brian McQueen Date: Thu, 8 Jan 2015 00:41:27 -0800 Subject: [PATCH 63/67] add server options to set redis and cassandra in the filer and corrected filer option description text --- go/weed/server.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go/weed/server.go b/go/weed/server.go index 062b02477a..0b973f7e16 100644 --- a/go/weed/server.go +++ b/go/weed/server.go @@ -81,6 +81,10 @@ func init() { filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified") filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.") filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") + filerOptions.cassandra_server = cmdFiler.Flag.String("filer.cassandra.server", "", "host[:port] of the cassandra server") + filerOptions.cassandra_keyspace = cmdFiler.Flag.String("filer.cassandra.keyspace", "seaweed", "keyspace of the cassandra server") + filerOptions.redis_server = cmdServer.Flag.String("filer.redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379") + filerOptions.redis_database = cmdFiler.Flag.Int("filer.redis.database", 0, "the database on the redis server") } From e7741db9a27f5be35bb28dc65c013f05541590c4 Mon Sep 17 00:00:00 2001 From: Brian McQueen Date: Thu, 8 Jan 2015 00:44:41 -0800 Subject: [PATCH 64/67] fixed description error in redis.database --- go/weed/filer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/weed/filer.go b/go/weed/filer.go index bc528be33e..5b3fd2b677 100644 --- a/go/weed/filer.go +++ b/go/weed/filer.go @@ -38,7 +38,7 @@ func init() { f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server") f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server") - f.redis_server = cmdFiler.Flag.String("redis.server", "", "host:port of the cassandra server, e.g., 127.0.0.1:6379") + f.redis_server = cmdFiler.Flag.String("redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379") f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server") } From 7304c840e3f25c0e59f12a708d82da6223c4d1bd Mon Sep 17 00:00:00 2001 From: yanyiwu Date: Thu, 8 Jan 2015 16:15:04 +0800 Subject: [PATCH 65/67] update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 46f3c04304..ab4f54fd55 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ weed +tags +*.swp From fe91fd00ad18d1a1758cee14fb929d512c0bc415 Mon Sep 17 00:00:00 2001 From: yanyiwu Date: Thu, 8 Jan 2015 19:04:10 +0800 Subject: [PATCH 66/67] fix bug: 'http: multiple response.WriteHeader calls' --- go/weed/weed_server/common.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index 3968f3191b..44ffcce47e 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -65,7 +65,6 @@ func writeJsonQuiet(w http.ResponseWriter, r *http.Request, httpStatus int, obj } } func writeJsonError(w http.ResponseWriter, r *http.Request, httpStatus int, err error) { - w.WriteHeader(http.StatusInternalServerError) m := make(map[string]interface{}) m["error"] = err.Error() writeJsonQuiet(w, r, httpStatus, m) From 543f5af5af796ebb9e1beb69b01292104f191d2e Mon Sep 17 00:00:00 2001 From: yanyiwu Date: Fri, 9 Jan 2015 19:24:49 +0800 Subject: [PATCH 67/67] add w.Write for FilerServer HTTP response --- go/weed/weed_server/filer_server_handlers.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go index 29a3d4b079..0811b79738 100644 --- a/go/weed/weed_server/filer_server_handlers.go +++ b/go/weed/weed_server/filer_server_handlers.go @@ -185,6 +185,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { return } w.WriteHeader(http.StatusCreated) + w.Write(resp_body) } // curl -X DELETE http://localhost:8888/path/to