Translation update

* origin/pr/1097:
  Revert "Cache file descriptors" (fixes #1096)

.mailmap

@@ -0,0 +1 @@
+NICKS

Godeps/Godeps.json

@@ -6,8 +6,8 @@
 	],
 	"Deps": [
 		{
-			"ImportPath": "github.com/AudriusButkevicius/lrufdcache",
-			"Rev": "9bddff8f67224ab3e7d80525a6ae9bcf1ce10769"
+			"ImportPath": "github.com/AudriusButkevicius/lfu-go",
+			"Rev": "164bcecceb92fd6037f4d18a8d97b495ec6ef669"
 		},
 		{
 			"ImportPath": "github.com/bkaradzic/go-lz4",
@@ -25,10 +25,6 @@
 			"ImportPath": "github.com/calmh/xdr",
 			"Rev": "45c46b7db7ff83b8b9ee09bbd95f36ab50043ece"
 		},
-		{
-			"ImportPath": "github.com/golang/groupcache/lru",
-			"Rev": "f391194b967ae0d21deadc861ea87120d9687447"
-		},
 		{
 			"ImportPath": "github.com/juju/ratelimit",
 			"Rev": "f9f36d11773655c0485207f0ad30dc2655f69d56"

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2012 Dave Grijalva
+
+Permission is hereby granted, free of charge, to any person obtaining a 
+copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation 
+the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the 
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included 
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
+THE SOFTWARE.

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/README.md

@@ -0,0 +1,19 @@
+A simple LFU cache for golang.  Based on the paper [An O(1) algorithm for implementing the LFU cache eviction scheme](http://dhruvbird.com/lfu.pdf).
+
+Usage:
+
+```go
+import "github.com/dgrijalva/lfu-go"
+
+// Make a new thing
+c := lfu.New()
+
+// Set some values
+c.Set("myKey", myValue)
+
+// Retrieve some values
+myValue = c.Get("myKey")
+
+// Evict some values
+c.Evict(1)
+```

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/lfu.go

@@ -0,0 +1,156 @@
+package lfu
+
+import (
+	"container/list"
+	"sync"
+)
+
+type Eviction struct {
+	Key   string
+	Value interface{}
+}
+
+type Cache struct {
+	// If len > UpperBound, cache will automatically evict
+	// down to LowerBound.  If either value is 0, this behavior
+	// is disabled.
+	UpperBound      int
+	LowerBound      int
+	values          map[string]*cacheEntry
+	freqs           *list.List
+	len             int
+	lock            *sync.Mutex
+	EvictionChannel chan<- Eviction
+}
+
+type cacheEntry struct {
+	key      string
+	value    interface{}
+	freqNode *list.Element
+}
+
+type listEntry struct {
+	entries map[*cacheEntry]byte
+	freq    int
+}
+
+func New() *Cache {
+	c := new(Cache)
+	c.values = make(map[string]*cacheEntry)
+	c.freqs = list.New()
+	c.lock = new(sync.Mutex)
+	return c
+}
+
+func (c *Cache) Get(key string) interface{} {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if e, ok := c.values[key]; ok {
+		c.increment(e)
+		return e.value
+	}
+	return nil
+}
+
+func (c *Cache) Set(key string, value interface{}) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if e, ok := c.values[key]; ok {
+		// value already exists for key.  overwrite
+		e.value = value
+		c.increment(e)
+	} else {
+		// value doesn't exist.  insert
+		e := new(cacheEntry)
+		e.key = key
+		e.value = value
+		c.values[key] = e
+		c.increment(e)
+		c.len++
+		// bounds mgmt
+		if c.UpperBound > 0 && c.LowerBound > 0 {
+			if c.len > c.UpperBound {
+				c.evict(c.len - c.LowerBound)
+			}
+		}
+	}
+}
+
+func (c *Cache) Len() int {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.len
+}
+
+func (c *Cache) Evict(count int) int {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.evict(count)
+}
+
+func (c *Cache) evict(count int) int {
+	// No lock here so it can be called
+	// from within the lock (during Set)
+	var evicted int
+	for i := 0; i < count; {
+		if place := c.freqs.Front(); place != nil {
+			for entry, _ := range place.Value.(*listEntry).entries {
+				if i < count {
+					if c.EvictionChannel != nil {
+						c.EvictionChannel <- Eviction{
+							Key:   entry.key,
+							Value: entry.value,
+						}
+					}
+					delete(c.values, entry.key)
+					c.remEntry(place, entry)
+					evicted++
+					c.len--
+					i++
+				}
+			}
+		}
+	}
+	return evicted
+}
+
+func (c *Cache) increment(e *cacheEntry) {
+	currentPlace := e.freqNode
+	var nextFreq int
+	var nextPlace *list.Element
+	if currentPlace == nil {
+		// new entry
+		nextFreq = 1
+		nextPlace = c.freqs.Front()
+	} else {
+		// move up
+		nextFreq = currentPlace.Value.(*listEntry).freq + 1
+		nextPlace = currentPlace.Next()
+	}
+
+	if nextPlace == nil || nextPlace.Value.(*listEntry).freq != nextFreq {
+		// create a new list entry
+		li := new(listEntry)
+		li.freq = nextFreq
+		li.entries = make(map[*cacheEntry]byte)
+		if currentPlace != nil {
+			nextPlace = c.freqs.InsertAfter(li, currentPlace)
+		} else {
+			nextPlace = c.freqs.PushFront(li)
+		}
+	}
+	e.freqNode = nextPlace
+	nextPlace.Value.(*listEntry).entries[e] = 1
+	if currentPlace != nil {
+		// remove from current position
+		c.remEntry(currentPlace, e)
+	}
+}
+
+func (c *Cache) remEntry(place *list.Element, entry *cacheEntry) {
+	entries := place.Value.(*listEntry).entries
+	delete(entries, entry)
+	if len(entries) == 0 {
+		c.freqs.Remove(place)
+	}
+}

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/lfu_test.go

@@ -0,0 +1,68 @@
+package lfu
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestLFU(t *testing.T) {
+	c := New()
+	c.Set("a", "a")
+	if v := c.Get("a"); v != "a" {
+		t.Errorf("Value was not saved: %v != 'a'", v)
+	}
+	if l := c.Len(); l != 1 {
+		t.Errorf("Length was not updated: %v != 1", l)
+	}
+
+	c.Set("b", "b")
+	if v := c.Get("b"); v != "b" {
+		t.Errorf("Value was not saved: %v != 'b'", v)
+	}
+	if l := c.Len(); l != 2 {
+		t.Errorf("Length was not updated: %v != 2", l)
+	}
+
+	c.Get("a")
+	evicted := c.Evict(1)
+	if v := c.Get("a"); v != "a" {
+		t.Errorf("Value was improperly evicted: %v != 'a'", v)
+	}
+	if v := c.Get("b"); v != nil {
+		t.Errorf("Value was not evicted: %v", v)
+	}
+	if l := c.Len(); l != 1 {
+		t.Errorf("Length was not updated: %v != 1", l)
+	}
+	if evicted != 1 {
+		t.Errorf("Number of evicted items is wrong: %v != 1", evicted)
+	}
+}
+
+func TestBoundsMgmt(t *testing.T) {
+	c := New()
+	c.UpperBound = 10
+	c.LowerBound = 5
+
+	for i := 0; i < 100; i++ {
+		c.Set(fmt.Sprintf("%v", i), i)
+	}
+	if c.Len() > 10 {
+		t.Errorf("Bounds management failed to evict properly: %v", c.Len())
+	}
+}
+
+func TestEviction(t *testing.T) {
+	ch := make(chan Eviction, 1)
+
+	c := New()
+	c.EvictionChannel = ch
+	c.Set("a", "b")
+	c.Evict(1)
+
+	ev := <-ch
+
+	if ev.Key != "a" || ev.Value.(string) != "b" {
+		t.Error("Incorrect item")
+	}
+}

Godeps/_workspace/src/github.com/AudriusButkevicius/lrufdcache/.gitignore

@@ -1,24 +0,0 @@
-# Compiled Object files, Static and Dynamic libs (Shared Objects)
-*.o
-*.a
-*.so
-
-# Folders
-_obj
-_test
-
-# Architecture specific extensions/prefixes
-*.[568vq]
-[568vq].out
-
-*.cgo1.go
-*.cgo2.c
-_cgo_defun.c
-_cgo_gotypes.go
-_cgo_export.*
-
-_testmain.go
-
-*.exe
-*.test
-*.prof

Godeps/_workspace/src/github.com/AudriusButkevicius/lrufdcache/LICENSE

@@ -1,25 +0,0 @@
-This is free and unencumbered software released into the public domain.
-
-Anyone is free to copy, modify, publish, use, compile, sell, or
-distribute this software, either in source code form or as a compiled
-binary, for any purpose, commercial or non-commercial, and by any
-means.
-
-In jurisdictions that recognize copyright laws, the author or authors
-of this software dedicate any and all copyright interest in the
-software to the public domain. We make this dedication for the benefit
-of the public at large and to the detriment of our heirs and
-successors. We intend this dedication to be an overt act of
-relinquishment in perpetuity of all present and future rights to this
-software under copyright law.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.
-
-For more information, please refer to <http://unlicense.org>
-

Godeps/_workspace/src/github.com/AudriusButkevicius/lrufdcache/README.md

@@ -1,4 +0,0 @@
-lrufdcache
-==========
-
-A LRU file descriptor cache

Godeps/_workspace/src/github.com/AudriusButkevicius/lrufdcache/lrufdcache.go

@@ -1,88 +0,0 @@
-// Package logger implements a LRU file descriptor cache for concurrent ReadAt
-// calls.
-package lrufdcache
-
-import (
-	"os"
-	"sync"
-
-	"github.com/golang/groupcache/lru"
-)
-
-// A wrapper around *os.File which counts references
-type CachedFile struct {
-	file *os.File
-	wg   sync.WaitGroup
-	// Locking between file.Close and file.ReadAt
-	// (just to please the race detector...)
-	flock sync.RWMutex
-}
-
-// Tells the cache that we are done using the file, but it's up to the cache
-// to decide when this file will really be closed. The error, if any, will be
-// lost.
-func (f *CachedFile) Close() error {
-	f.wg.Done()
-	return nil
-}
-
-// Read the file at the given offset.
-func (f *CachedFile) ReadAt(buf []byte, at int64) (int, error) {
-	f.flock.RLock()
-	defer f.flock.RUnlock()
-	return f.file.ReadAt(buf, at)
-}
-
-type FileCache struct {
-	cache *lru.Cache
-	mut   sync.Mutex
-}
-
-// Create a new cache with the number of entries to hold.
-func NewCache(entries int) *FileCache {
-	c := FileCache{
-		cache: lru.New(entries),
-	}
-
-	c.cache.OnEvicted = func(key lru.Key, fdi interface{}) {
-		// The file might not have been closed by all openers yet, therefore
-		// spawn a routine which waits for that to happen and then closes the
-		// file.
-		go func(item *CachedFile) {
-			item.wg.Wait()
-			item.flock.Lock()
-			item.file.Close()
-			item.flock.Unlock()
-		}(fdi.(*CachedFile))
-	}
-	return &c
-}
-
-// Open and cache a file descriptor or use an existing cached descriptor for
-// the given path.
-func (c *FileCache) Open(path string) (*CachedFile, error) {
-	// Evictions can only happen during c.cache.Add, and there is a potential
-	// race between c.cache.Get and cfd.wg.Add where if not guarded by a mutex
-	// could result in cfd getting closed before the counter is incremented if
-	// a concurrent routine does a c.cache.Add
-	c.mut.Lock()
-	defer c.mut.Unlock()
-	fdi, ok := c.cache.Get(path)
-	if ok {
-		cfd := fdi.(*CachedFile)
-		cfd.wg.Add(1)
-		return cfd, nil
-	}
-
-	fd, err := os.Open(path)
-	if err != nil {
-		return nil, err
-	}
-	cfd := &CachedFile{
-		file: fd,
-		wg:   sync.WaitGroup{},
-	}
-	cfd.wg.Add(1)
-	c.cache.Add(path, cfd)
-	return cfd, nil
-}

Godeps/_workspace/src/github.com/AudriusButkevicius/lrufdcache/lrufdcache_test.go

@@ -1,195 +0,0 @@
-package lrufdcache
-
-import (
-	"io/ioutil"
-	"os"
-	"sync"
-	"time"
-
-	"testing"
-)
-
-func TestNoopReadFailsOnClosed(t *testing.T) {
-	fd, err := ioutil.TempFile("", "fdcache")
-	if err != nil {
-		t.Fatal(err)
-		return
-	}
-	fd.WriteString("test")
-	fd.Close()
-	buf := make([]byte, 4)
-	defer os.Remove(fd.Name())
-
-	_, err = fd.ReadAt(buf, 0)
-	if err == nil {
-		t.Fatal("Expected error")
-	}
-}
-
-func TestSingleFileEviction(t *testing.T) {
-	c := NewCache(1)
-
-	wg := sync.WaitGroup{}
-
-	fd, err := ioutil.TempFile("", "fdcache")
-	if err != nil {
-		t.Fatal(err)
-		return
-	}
-	fd.WriteString("test")
-	fd.Close()
-	buf := make([]byte, 4)
-	defer os.Remove(fd.Name())
-
-	for k := 0; k < 100; k++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
-			cfd, err := c.Open(fd.Name())
-			if err != nil {
-				t.Fatal(err)
-				return
-			}
-			defer cfd.Close()
-
-			_, err = cfd.ReadAt(buf, 0)
-			if err != nil {
-				t.Fatal(err)
-			}
-		}()
-	}
-
-	wg.Wait()
-}
-
-func TestMultifileEviction(t *testing.T) {
-	c := NewCache(1)
-
-	wg := sync.WaitGroup{}
-
-	for k := 0; k < 100; k++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
-			fd, err := ioutil.TempFile("", "fdcache")
-			if err != nil {
-				t.Fatal(err)
-				return
-			}
-			fd.WriteString("test")
-			fd.Close()
-			buf := make([]byte, 4)
-			defer os.Remove(fd.Name())
-
-			cfd, err := c.Open(fd.Name())
-			if err != nil {
-				t.Fatal(err)
-				return
-			}
-			defer cfd.Close()
-
-			_, err = cfd.ReadAt(buf, 0)
-			if err != nil {
-				t.Fatal(err)
-			}
-		}()
-	}
-
-	wg.Wait()
-}
-
-func TestMixedEviction(t *testing.T) {
-	c := NewCache(1)
-
-	wg := sync.WaitGroup{}
-	wg2 := sync.WaitGroup{}
-	for i := 0; i < 100; i++ {
-		wg2.Add(1)
-		go func() {
-			defer wg2.Done()
-			fd, err := ioutil.TempFile("", "fdcache")
-			if err != nil {
-				t.Fatal(err)
-				return
-			}
-			fd.WriteString("test")
-			fd.Close()
-			buf := make([]byte, 4)
-
-			for k := 0; k < 100; k++ {
-				wg.Add(1)
-				go func() {
-					defer wg.Done()
-
-					cfd, err := c.Open(fd.Name())
-					if err != nil {
-						t.Fatal(err)
-						return
-					}
-					defer cfd.Close()
-
-					_, err = cfd.ReadAt(buf, 0)
-					if err != nil {
-						t.Fatal(err)
-					}
-				}()
-			}
-		}()
-	}
-
-	wg2.Wait()
-	wg.Wait()
-}
-
-func TestLimit(t *testing.T) {
-	testcase := 50
-	fd, err := ioutil.TempFile("", "fdcache")
-	if err != nil {
-		t.Fatal(err)
-		return
-	}
-	fd.Close()
-	defer os.Remove(fd.Name())
-
-	c := NewCache(testcase)
-	fds := make([]*CachedFile, testcase*2)
-	for i := 0; i < testcase*2; i++ {
-		fd, err := ioutil.TempFile("", "fdcache")
-		if err != nil {
-			t.Fatal(err)
-			return
-		}
-		fd.WriteString("test")
-		fd.Close()
-		defer os.Remove(fd.Name())
-
-		nfd, err := c.Open(fd.Name())
-		if err != nil {
-			t.Fatal(err)
-			return
-		}
-		fds = append(fds, nfd)
-		nfd.Close()
-	}
-
-	// Allow closes to happen
-	time.Sleep(time.Millisecond * 100)
-
-	buf := make([]byte, 4)
-	ok := 0
-	for _, fd := range fds {
-		if fd == nil {
-			continue
-		}
-		_, err := fd.ReadAt(buf, 0)
-		if err == nil {
-			ok++
-		}
-	}
-
-	if ok > testcase {
-		t.Fatal("More than", testcase, "fds open")
-	}
-}

Godeps/_workspace/src/github.com/golang/groupcache/lru/lru.go

@@ -1,121 +0,0 @@
-/*
-Copyright 2013 Google Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package lru implements an LRU cache.
-package lru
-
-import "container/list"
-
-// Cache is an LRU cache. It is not safe for concurrent access.
-type Cache struct {
-	// MaxEntries is the maximum number of cache entries before
-	// an item is evicted. Zero means no limit.
-	MaxEntries int
-
-	// OnEvicted optionally specificies a callback function to be
-	// executed when an entry is purged from the cache.
-	OnEvicted func(key Key, value interface{})
-
-	ll    *list.List
-	cache map[interface{}]*list.Element
-}
-
-// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators
-type Key interface{}
-
-type entry struct {
-	key   Key
-	value interface{}
-}
-
-// New creates a new Cache.
-// If maxEntries is zero, the cache has no limit and it's assumed
-// that eviction is done by the caller.
-func New(maxEntries int) *Cache {
-	return &Cache{
-		MaxEntries: maxEntries,
-		ll:         list.New(),
-		cache:      make(map[interface{}]*list.Element),
-	}
-}
-
-// Add adds a value to the cache.
-func (c *Cache) Add(key Key, value interface{}) {
-	if c.cache == nil {
-		c.cache = make(map[interface{}]*list.Element)
-		c.ll = list.New()
-	}
-	if ee, ok := c.cache[key]; ok {
-		c.ll.MoveToFront(ee)
-		ee.Value.(*entry).value = value
-		return
-	}
-	ele := c.ll.PushFront(&entry{key, value})
-	c.cache[key] = ele
-	if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries {
-		c.RemoveOldest()
-	}
-}
-
-// Get looks up a key's value from the cache.
-func (c *Cache) Get(key Key) (value interface{}, ok bool) {
-	if c.cache == nil {
-		return
-	}
-	if ele, hit := c.cache[key]; hit {
-		c.ll.MoveToFront(ele)
-		return ele.Value.(*entry).value, true
-	}
-	return
-}
-
-// Remove removes the provided key from the cache.
-func (c *Cache) Remove(key Key) {
-	if c.cache == nil {
-		return
-	}
-	if ele, hit := c.cache[key]; hit {
-		c.removeElement(ele)
-	}
-}
-
-// RemoveOldest removes the oldest item from the cache.
-func (c *Cache) RemoveOldest() {
-	if c.cache == nil {
-		return
-	}
-	ele := c.ll.Back()
-	if ele != nil {
-		c.removeElement(ele)
-	}
-}
-
-func (c *Cache) removeElement(e *list.Element) {
-	c.ll.Remove(e)
-	kv := e.Value.(*entry)
-	delete(c.cache, kv.key)
-	if c.OnEvicted != nil {
-		c.OnEvicted(kv.key, kv.value)
-	}
-}
-
-// Len returns the number of items in the cache.
-func (c *Cache) Len() int {
-	if c.cache == nil {
-		return 0
-	}
-	return c.ll.Len()
-}

Godeps/_workspace/src/github.com/golang/groupcache/lru/lru_test.go

@@ -1,73 +0,0 @@
-/*
-Copyright 2013 Google Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package lru
-
-import (
-	"testing"
-)
-
-type simpleStruct struct {
-	int
-	string
-}
-
-type complexStruct struct {
-	int
-	simpleStruct
-}
-
-var getTests = []struct {
-	name       string
-	keyToAdd   interface{}
-	keyToGet   interface{}
-	expectedOk bool
-}{
-	{"string_hit", "myKey", "myKey", true},
-	{"string_miss", "myKey", "nonsense", false},
-	{"simple_struct_hit", simpleStruct{1, "two"}, simpleStruct{1, "two"}, true},
-	{"simeple_struct_miss", simpleStruct{1, "two"}, simpleStruct{0, "noway"}, false},
-	{"complex_struct_hit", complexStruct{1, simpleStruct{2, "three"}},
-		complexStruct{1, simpleStruct{2, "three"}}, true},
-}
-
-func TestGet(t *testing.T) {
-	for _, tt := range getTests {
-		lru := New(0)
-		lru.Add(tt.keyToAdd, 1234)
-		val, ok := lru.Get(tt.keyToGet)
-		if ok != tt.expectedOk {
-			t.Fatalf("%s: cache hit = %v; want %v", tt.name, ok, !ok)
-		} else if ok && val != 1234 {
-			t.Fatalf("%s expected get to return 1234 but got %v", tt.name, val)
-		}
-	}
-}
-
-func TestRemove(t *testing.T) {
-	lru := New(0)
-	lru.Add("myKey", 1234)
-	if val, ok := lru.Get("myKey"); !ok {
-		t.Fatal("TestRemove returned no match")
-	} else if val != 1234 {
-		t.Fatalf("TestRemove failed.  Expected %d, got %v", 1234, val)
-	}
-
-	lru.Remove("myKey")
-	if _, ok := lru.Get("myKey"); ok {
-		t.Fatal("TestRemove returned a removed entry")
-	}
-}

NICKS

@@ -0,0 +1,34 @@
+# This file maps email addresses used in commits to nicks used the changelog.
+
+AudriusButkevicius	<audrius.butkevicius@gmail.com>
+KayoticSully	<kayoticsully@gmail.com>
+Nutomic		<me@nutomic.com>
+Vilbrekin	<vilbrekin@gmail.com>
+Zillode		<zillode@zillode.be>
+alex2108	<register-github@alex-graf.de>
+andrew-d	<andrew@du.nham.ca>
+asdil12		<dominik@heidler.eu>
+bigbear2nd	<bigbear2nd@gmail.com>
+bsidhom		<bsidhom@gmail.com>
+calmh		<jakob@nym.se>
+cdata		<chris@scriptolo.gy>
+ceh		<emil@hessman.se>
+cqcallaw	<enlightened.despot@gmail.com>
+filoozoom	<philippe@schommers.be>
+frioux		<frew@afoolishmanifesto.com> <frioux@gmail.com>
+gillisig	<gilli@vx.is>
+jedie		<github.com@jensdiemer.de> <git@jensdiemer.de>
+jpjp		<jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
+kozec		<kozec@kozec.com>
+marcindziadus	<dziadus.marcin@gmail.com>
+mvdan		<mvdan@mvdan.cc>
+philips		<brandon@ifup.org>
+piobpl		<piotrb10@gmail.com>
+pluby		<phill.luby@newredo.com>
+pyfisch		<pyfisch@gmail.com>
+qbit		<qbit@deftly.net>
+seehuhn		<voss@seehuhn.de>
+snnd		<dw@risu.io>
+tojrobinson	<tully@tojr.org>
+uok		<ueomkail@gmail.com>
+veeti		<veeti.paananen@rojekti.fi>

changelog.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+since="$1"
+if [[ -z $since ]] ; then
+	since="$(git describe --abbrev=0 HEAD^).."
+fi
+
+git log --pretty=format:'* %s (%h, @%aN)' "$since" | egrep 'fixes #\d|ref #\d'
+

cmd/syncthing/gui.go

@@ -353,7 +353,7 @@ func restPostConfig(m *model.Model, w http.ResponseWriter, r *http.Request) {
 		if curAcc := cfg.Options().URAccepted; newCfg.Options.URAccepted > curAcc {
 			// UR was enabled
 			newCfg.Options.URAccepted = usageReportVersion
-			newCfg.Options.URUniqueID = randomString(6)
+			newCfg.Options.URUniqueID = randomString(8)
 			err := sendUsageReport(m)
 			if err != nil {
 				l.Infoln("Usage report:", err)

cmd/syncthing/gui_csrf.go

@@ -17,8 +17,6 @@ package main
 
 import (
 	"bufio"
-	"crypto/rand"
-	"encoding/base64"
 	"fmt"
 	"net/http"
 	"os"
@@ -88,7 +86,7 @@ func validCsrfToken(token string) bool {
 }
 
 func newCsrfToken() string {
-	token := randomString(30)
+	token := randomString(32)
 
 	csrfMut.Lock()
 	csrfTokens = append(csrfTokens, token)
@@ -140,13 +138,3 @@ func loadCsrfTokens() {
 		csrfTokens = append(csrfTokens, s.Text())
 	}
 }
-
-func randomString(len int) string {
-	bs := make([]byte, len)
-	_, err := rand.Reader.Read(bs)
-	if err != nil {
-		l.Fatalln(err)
-	}
-
-	return base64.StdEncoding.EncodeToString(bs)
-}

cmd/syncthing/main.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"io"
 	"log"
-	"math/rand"
 	"net"
 	"net/http"
 	_ "net/http/pprof"
@@ -177,10 +176,6 @@ are mostly useful for developers. Use with care.
                available CPU cores.`
 )
 
-func init() {
-	rand.Seed(time.Now().UnixNano())
-}
-
 // Command line and environment options
 var (
 	reset             bool
@@ -383,6 +378,10 @@ func syncthingMain() {
 		}
 	}
 
+	// We reinitialize the predictable RNG with our device ID, to get a
+	// sequence that is always the same but unique to this syncthing instance.
+	predictableRandom.Seed(seedFromBytes(cert.Certificate[0]))
+
 	myID = protocol.NewDeviceID(cert.Certificate[0])
 	l.SetPrefix(fmt.Sprintf("[%s] ", myID.String()[:5]))
 
@@ -574,7 +573,9 @@ func syncthingMain() {
 		if opts.URUniqueID == "" {
 			// Previously the ID was generated from the node ID. We now need
 			// to generate a new one.
-			opts.URUniqueID = randomString(6)
+			opts.URUniqueID = randomString(8)
+			cfg.SetOptions(opts)
+			cfg.Save()
 		}
 		go usageReportingLoop(m)
 		go func() {
@@ -779,11 +780,8 @@ func setupExternalPort(igd *upnp.IGD, port int) int {
 		return 0
 	}
 
-	// We seed the random number generator with the node ID to get a
-	// repeatable sequence of random external ports.
-	rnd := rand.NewSource(certSeed(cert.Certificate[0]))
 	for i := 0; i < 10; i++ {
-		r := 1024 + int(rnd.Int63()%(65535-1024))
+		r := 1024 + predictableRandom.Intn(65535-1024)
 		err := igd.AddPortMapping(upnp.TCP, r, port, "syncthing", cfg.Options().UPnPLease*60)
 		if err == nil {
 			return r

cmd/syncthing/random.go

@@ -0,0 +1,67 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"crypto/md5"
+	cryptoRand "crypto/rand"
+	"encoding/binary"
+	mathRand "math/rand"
+)
+
+// randomCharset contains the characters that can make up a randomString().
+const randomCharset = "01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-"
+
+// predictableRandom is an RNG that will always have the same sequence. It
+// will be seeded with the device ID during startup, so that the sequence is
+// predictable but varies between instances.
+var predictableRandom = mathRand.New(mathRand.NewSource(42))
+
+func init() {
+	// The default RNG should be seeded with something good.
+	mathRand.Seed(randomInt64())
+}
+
+// randomString returns a string of random characters (taken from
+// randomCharset) of the specified length.
+func randomString(l int) string {
+	bs := make([]byte, l)
+	for i := range bs {
+		bs[i] = randomCharset[mathRand.Intn(len(randomCharset))]
+	}
+	return string(bs)
+}
+
+// randomInt64 returns a strongly random int64, slowly
+func randomInt64() int64 {
+	var bs [8]byte
+	n, err := cryptoRand.Reader.Read(bs[:])
+	if n != 8 || err != nil {
+		panic("randomness failure")
+	}
+	return seedFromBytes(bs[:])
+}
+
+// seedFromBytes calculates a weak 64 bit hash from the given byte slice,
+// suitable for use a predictable random seed.
+func seedFromBytes(bs []byte) int64 {
+	h := md5.New()
+	h.Write(bs)
+	s := h.Sum(nil)
+	// The MD5 hash of the byte slice is 16 bytes long. We interpret it as two
+	// uint64s and XOR them together.
+	return int64(binary.BigEndian.Uint64(s[0:]) ^ binary.BigEndian.Uint64(s[8:]))
+}

cmd/syncthing/random_test.go

@@ -0,0 +1,80 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import "testing"
+
+func TestPredictableRandom(t *testing.T) {
+	// predictable random sequence is predictable
+	e := 3440579354231278675
+	if v := predictableRandom.Int(); v != e {
+		t.Errorf("Unexpected random value %d != %d", v, e)
+	}
+}
+
+func TestSeedFromBytes(t *testing.T) {
+	// should always return the same seed for the same bytes
+	tcs := []struct {
+		bs []byte
+		v  int64
+	}{
+		{[]byte("hello world"), -3639725434188061933},
+		{[]byte("hello worlx"), -2539100776074091088},
+	}
+
+	for _, tc := range tcs {
+		if v := seedFromBytes(tc.bs); v != tc.v {
+			t.Errorf("Unexpected seed value %d != %d", v, tc.v)
+		}
+	}
+}
+
+func TestRandomString(t *testing.T) {
+	for _, l := range []int{0, 1, 2, 3, 4, 8, 42} {
+		s := randomString(l)
+		if len(s) != l {
+			t.Errorf("Incorrect length %d != %s", len(s), l)
+		}
+	}
+
+	strings := make([]string, 1000)
+	for i := range strings {
+		strings[i] = randomString(8)
+		for j := range strings {
+			if i == j {
+				continue
+			}
+			if strings[i] == strings[j] {
+				t.Errorf("Repeated random string %q", strings[i])
+			}
+		}
+	}
+}
+
+func TestRandomInt64(t *testing.T) {
+	ints := make([]int64, 1000)
+	for i := range ints {
+		ints[i] = randomInt64()
+		for j := range ints {
+			if i == j {
+				continue
+			}
+			if ints[i] == ints[j] {
+				t.Errorf("Repeated random int64 %d", ints[i])
+			}
+		}
+	}
+}

cmd/syncthing/tls.go

@@ -19,11 +19,9 @@ import (
 	"bufio"
 	"crypto/rand"
 	"crypto/rsa"
-	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
-	"encoding/binary"
 	"encoding/pem"
 	"io"
 	"math/big"
@@ -45,13 +43,6 @@ func loadCert(dir string, prefix string) (tls.Certificate, error) {
 	return tls.LoadX509KeyPair(cf, kf)
 }
 
-func certSeed(bs []byte) int64 {
-	hf := sha256.New()
-	hf.Write(bs)
-	id := hf.Sum(nil)
-	return int64(binary.BigEndian.Uint64(id))
-}
-
 func newCertificate(dir string, prefix string) {
 	l.Infoln("Generating RSA key and certificate...")
 

gui/assets/lang/lang-bg.json

@@ -8,7 +8,7 @@
     "Allow Anonymous Usage Reporting?": "Разреши анонимен доклад за ползване на програмата?",
     "Anonymous Usage Reporting": "Анонимен Доклад",
     "Any devices configured on an introducer device will be added to this device as well.": "Устройства настроени на introducer компютъра също ще бъдат добавени към този компютър.",
-    "Automatic upgrades": "Automatic upgrades",
+    "Automatic upgrades": "Автоматични ъпдейти",
     "Bugs": "Бъгове",
     "CPU Utilization": "Натоварване на Процесора",
     "Close": "Затвори",

internal/model/model.go

@@ -41,8 +41,6 @@ import (
 	"github.com/syncthing/syncthing/internal/stats"
 	"github.com/syncthing/syncthing/internal/symlinks"
 	"github.com/syncthing/syncthing/internal/versioner"
-
-	"github.com/AudriusButkevicius/lrufdcache"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
@@ -88,7 +86,6 @@ type Model struct {
 	db              *leveldb.DB
 	finder          *files.BlockFinder
 	progressEmitter *ProgressEmitter
-	cache           *lrufdcache.FileCache
 
 	deviceName    string
 	clientName    string
@@ -130,7 +127,6 @@ func NewModel(cfg *config.ConfigWrapper, deviceName, clientName, clientVersion s
 	m := &Model{
 		cfg:                cfg,
 		db:                 db,
-		cache:              lrufdcache.NewCache(25),
 		deviceName:         deviceName,
 		clientName:         clientName,
 		clientVersion:      clientVersion,
@@ -576,7 +572,7 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 					l.Infof("Adding device %v to config (vouched for by introducer %v)", id, deviceID)
 					newDeviceCfg := config.DeviceConfiguration{
 						DeviceID:    id,
-						Compression: true,
+						Compression: m.cfg.Devices()[deviceID].Compression,
 						Addresses:   []string{"dynamic"},
 					}
 
@@ -699,11 +695,12 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 		}
 		reader = strings.NewReader(target)
 	} else {
-		reader, err = m.cache.Open(fn)
+		reader, err = os.Open(fn) // XXX: Inefficient, should cache fd?
 		if err != nil {
 			return nil, err
 		}
-		defer reader.(*lrufdcache.CachedFile).Close()
+
+		defer reader.(*os.File).Close()
 	}
 
 	buf := make([]byte, size)

internal/model/puller.go

@@ -26,6 +26,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/AudriusButkevicius/lfu-go"
+
 	"github.com/syncthing/syncthing/internal/config"
 	"github.com/syncthing/syncthing/internal/events"
 	"github.com/syncthing/syncthing/internal/osutil"
@@ -600,6 +602,19 @@ nextFile:
 			p.progressEmitter.Register(state.sharedPullerState)
 		}
 
+		evictionChan := make(chan lfu.Eviction)
+
+		fdCache := lfu.New()
+		fdCache.UpperBound = 50
+		fdCache.LowerBound = 20
+		fdCache.EvictionChannel = evictionChan
+
+		go func() {
+			for item := range evictionChan {
+				item.Value.(*os.File).Close()
+			}
+		}()
+
 		folderRoots := make(map[string]string)
 		p.model.fmut.RLock()
 		for folder, cfg := range p.model.folderCfgs {
@@ -613,11 +628,18 @@ nextFile:
 			found := p.model.finder.Iterate(block.Hash, func(folder, file string, index uint32) bool {
 				path := filepath.Join(folderRoots[folder], file)
 
-				fd, err := p.model.cache.Open(path)
-				if err != nil {
-					return false
+				var fd *os.File
+
+				fdi := fdCache.Get(path)
+				if fdi != nil {
+					fd = fdi.(*os.File)
+				} else {
+					fd, err = os.Open(path)
+					if err != nil {
+						return false
+					}
+					fdCache.Set(path, fd)
 				}
-				defer fd.Close()
 
 				_, err = fd.ReadAt(buf, protocol.BlockSize*int64(index))
 				if err != nil {
@@ -666,6 +688,8 @@ nextFile:
 				state.copyDone()
 			}
 		}
+		fdCache.Evict(fdCache.Len())
+		close(evictionChan)
 		out <- state.sharedPullerState
 	}
 }