Do not crash on TLS connection without certificate

Still needs implemention in GUI

.gitignore

@@ -1,3 +1,7 @@
 syncthing
+syncthing.exe
+stcli
+stcli.exe
 *.tar.gz
-build
+*.zip
+*.asc

CONTRIBUTING.md

@@ -0,0 +1,54 @@
+Please do contribute! If you want to contribute but are unsure where to
+start, the [Contributions Needed
+page](https://github.com/calmh/syncthing/wiki/Contributions-Needed)
+lists areas in need of attention.
+
+## Licensing
+
+All contributions are made under the same MIT License as the rest of the
+project, except documentation which is licensed under the Creative
+Commons Attribution 4.0 International License. You retain the copyright
+to code you have written.
+
+When accepting your first contribution, the maintainer of the project
+will ensure that you are added to the CONTRIBUTORS file.
+
+## Building
+
+[See the wiki](https://github.com/calmh/syncthing/wiki/Building)
+
+## Branches
+
+ - `master` is the main branch containing good code that will end up in
+   the next release. You should base your work on it. It won't ever be
+   rebased or force-pushed to.
+
+ - `vx.y` branches exist to make patch releases on otherwise obsolete
+   minor releases. Should only contain fixes cherry picked from master.
+   Don't base any work on them.
+
+ - Other branches are probably topic branches and may be subject to
+   rebasing. Don't base any work on them unless you specifically know
+   otherwise.
+
+## Tags
+
+All releases are tagged semver style as `vx.y.z`. Release tags are
+signed by GPG key BCE524C7.
+
+## Tests
+
+Yes please!
+
+## Style
+
+`go fmt`
+
+## Documentation
+
+[Hack it here](https://github.com/calmh/syncthing/wiki)
+
+## License
+
+MIT
+

CONTRIBUTORS

@@ -0,0 +1,4 @@
+Aaron Bieber <qbit@deftly.net>
+Brandon Philips <brandon@ifup.org>
+James Patterson <jamespatterson@operamail.com>
+Philippe Schommers <philippe@schommers.be>

Godeps/Godeps.json

@@ -0,0 +1,36 @@
+{
+	"ImportPath": "github.com/calmh/syncthing",
+	"GoVersion": "go1.2.1",
+	"Packages": [
+		"./cmd/syncthing"
+	],
+	"Deps": [
+		{
+			"ImportPath": "code.google.com/p/go.text/transform",
+			"Comment": "null-81",
+			"Rev": "9cbe983aed9b0dfc73954433fead5e00866342ac"
+		},
+		{
+			"ImportPath": "code.google.com/p/go.text/unicode/norm",
+			"Comment": "null-81",
+			"Rev": "9cbe983aed9b0dfc73954433fead5e00866342ac"
+		},
+		{
+			"ImportPath": "github.com/calmh/ini",
+			"Rev": "386c4240a9684d91d9ec4d93651909b49c7269e1"
+		},
+		{
+			"ImportPath": "github.com/codegangsta/inject",
+			"Rev": "9aea7a2fa5b79ef7fc00f63a575e72df33b4e886"
+		},
+		{
+			"ImportPath": "github.com/codegangsta/martini",
+			"Comment": "v0.1-142-g8659df7",
+			"Rev": "8659df7a51aebe6c6120268cd5a8b4c34fa8441a"
+		},
+		{
+			"ImportPath": "github.com/juju/ratelimit",
+			"Rev": "cbaa435c80a9716e086f25d409344b26c4039358"
+		}
+	]
+}

Godeps/Readme

@@ -0,0 +1,5 @@
+This directory tree is generated automatically by godep.
+
+Please do not edit.
+
+See https://github.com/tools/godep for more information.

Godeps/_workspace/.gitignore

@@ -0,0 +1,2 @@
+/pkg
+/bin

Godeps/_workspace/src/code.google.com/p/go.text/transform/examples_test.go

@@ -0,0 +1,37 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package transform_test
+
+import (
+	"fmt"
+	"unicode"
+
+	"code.google.com/p/go.text/transform"
+	"code.google.com/p/go.text/unicode/norm"
+)
+
+func ExampleRemoveFunc() {
+	input := []byte(`tschüß; до свидания`)
+
+	b := make([]byte, len(input))
+
+	t := transform.RemoveFunc(unicode.IsSpace)
+	n, _, _ := t.Transform(b, input, true)
+	fmt.Println(string(b[:n]))
+
+	t = transform.RemoveFunc(func(r rune) bool {
+		return !unicode.Is(unicode.Latin, r)
+	})
+	n, _, _ = t.Transform(b, input, true)
+	fmt.Println(string(b[:n]))
+
+	n, _, _ = t.Transform(b, norm.NFD.Bytes(input), true)
+	fmt.Println(string(b[:n]))
+
+	// Output:
+	// tschüß;досвидания
+	// tschüß
+	// tschuß
+}

Godeps/_workspace/src/code.google.com/p/go.text/transform/transform.go

@@ -0,0 +1,496 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package transform provides reader and writer wrappers that transform the
+// bytes passing through as well as various transformations. Example
+// transformations provided by other packages include normalization and
+// conversion between character sets.
+package transform
+
+import (
+	"errors"
+	"io"
+	"unicode/utf8"
+)
+
+var (
+	// ErrShortDst means that the destination buffer was too short to
+	// receive all of the transformed bytes.
+	ErrShortDst = errors.New("transform: short destination buffer")
+
+	// ErrShortSrc means that the source buffer has insufficient data to
+	// complete the transformation.
+	ErrShortSrc = errors.New("transform: short source buffer")
+
+	// errInconsistentByteCount means that Transform returned success (nil
+	// error) but also returned nSrc inconsistent with the src argument.
+	errInconsistentByteCount = errors.New("transform: inconsistent byte count returned")
+
+	// errShortInternal means that an internal buffer is not large enough
+	// to make progress and the Transform operation must be aborted.
+	errShortInternal = errors.New("transform: short internal buffer")
+)
+
+// Transformer transforms bytes.
+type Transformer interface {
+	// Transform writes to dst the transformed bytes read from src, and
+	// returns the number of dst bytes written and src bytes read. The
+	// atEOF argument tells whether src represents the last bytes of the
+	// input.
+	//
+	// Callers should always process the nDst bytes produced and account
+	// for the nSrc bytes consumed before considering the error err.
+	//
+	// A nil error means that all of the transformed bytes (whether freshly
+	// transformed from src or left over from previous Transform calls)
+	// were written to dst. A nil error can be returned regardless of
+	// whether atEOF is true. If err is nil then nSrc must equal len(src);
+	// the converse is not necessarily true.
+	//
+	// ErrShortDst means that dst was too short to receive all of the
+	// transformed bytes. ErrShortSrc means that src had insufficient data
+	// to complete the transformation. If both conditions apply, then
+	// either error may be returned. Other than the error conditions listed
+	// here, implementations are free to report other errors that arise.
+	Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error)
+}
+
+// TODO: Do we require that a Transformer be reusable if it returns a nil error
+// or do we always require a reset after use?  Is Reset mandatory or optional?
+
+// Reader wraps another io.Reader by transforming the bytes read.
+type Reader struct {
+	r   io.Reader
+	t   Transformer
+	err error
+
+	// dst[dst0:dst1] contains bytes that have been transformed by t but
+	// not yet copied out via Read.
+	dst        []byte
+	dst0, dst1 int
+
+	// src[src0:src1] contains bytes that have been read from r but not
+	// yet transformed through t.
+	src        []byte
+	src0, src1 int
+
+	// transformComplete is whether the transformation is complete,
+	// regardless of whether or not it was successful.
+	transformComplete bool
+}
+
+const defaultBufSize = 4096
+
+// NewReader returns a new Reader that wraps r by transforming the bytes read
+// via t.
+func NewReader(r io.Reader, t Transformer) *Reader {
+	return &Reader{
+		r:   r,
+		t:   t,
+		dst: make([]byte, defaultBufSize),
+		src: make([]byte, defaultBufSize),
+	}
+}
+
+// Read implements the io.Reader interface.
+func (r *Reader) Read(p []byte) (int, error) {
+	n, err := 0, error(nil)
+	for {
+		// Copy out any transformed bytes and return the final error if we are done.
+		if r.dst0 != r.dst1 {
+			n = copy(p, r.dst[r.dst0:r.dst1])
+			r.dst0 += n
+			if r.dst0 == r.dst1 && r.transformComplete {
+				return n, r.err
+			}
+			return n, nil
+		} else if r.transformComplete {
+			return 0, r.err
+		}
+
+		// Try to transform some source bytes, or to flush the transformer if we
+		// are out of source bytes. We do this even if r.r.Read returned an error.
+		// As the io.Reader documentation says, "process the n > 0 bytes returned
+		// before considering the error".
+		if r.src0 != r.src1 || r.err != nil {
+			r.dst0 = 0
+			r.dst1, n, err = r.t.Transform(r.dst, r.src[r.src0:r.src1], r.err == io.EOF)
+			r.src0 += n
+
+			switch {
+			case err == nil:
+				if r.src0 != r.src1 {
+					r.err = errInconsistentByteCount
+				}
+				// The Transform call was successful; we are complete if we
+				// cannot read more bytes into src.
+				r.transformComplete = r.err != nil
+				continue
+			case err == ErrShortDst && r.dst1 != 0:
+				// Make room in dst by copying out, and try again.
+				continue
+			case err == ErrShortSrc && r.src1-r.src0 != len(r.src) && r.err == nil:
+				// Read more bytes into src via the code below, and try again.
+			default:
+				r.transformComplete = true
+				// The reader error (r.err) takes precedence over the
+				// transformer error (err) unless r.err is nil or io.EOF.
+				if r.err == nil || r.err == io.EOF {
+					r.err = err
+				}
+				continue
+			}
+		}
+
+		// Move any untransformed source bytes to the start of the buffer
+		// and read more bytes.
+		if r.src0 != 0 {
+			r.src0, r.src1 = 0, copy(r.src, r.src[r.src0:r.src1])
+		}
+		n, r.err = r.r.Read(r.src[r.src1:])
+		r.src1 += n
+	}
+}
+
+// TODO: implement ReadByte (and ReadRune??).
+
+// Writer wraps another io.Writer by transforming the bytes read.
+// The user needs to call Close to flush unwritten bytes that may
+// be buffered.
+type Writer struct {
+	w   io.Writer
+	t   Transformer
+	dst []byte
+
+	// src[:n] contains bytes that have not yet passed through t.
+	src []byte
+	n   int
+}
+
+// NewWriter returns a new Writer that wraps w by transforming the bytes written
+// via t.
+func NewWriter(w io.Writer, t Transformer) *Writer {
+	return &Writer{
+		w:   w,
+		t:   t,
+		dst: make([]byte, defaultBufSize),
+		src: make([]byte, defaultBufSize),
+	}
+}
+
+// Write implements the io.Writer interface. If there are not enough
+// bytes available to complete a Transform, the bytes will be buffered
+// for the next write. Call Close to convert the remaining bytes.
+func (w *Writer) Write(data []byte) (n int, err error) {
+	src := data
+	if w.n > 0 {
+		// Append bytes from data to the last remainder.
+		// TODO: limit the amount copied on first try.
+		n = copy(w.src[w.n:], data)
+		w.n += n
+		src = w.src[:w.n]
+	}
+	for {
+		nDst, nSrc, err := w.t.Transform(w.dst, src, false)
+		if _, werr := w.w.Write(w.dst[:nDst]); werr != nil {
+			return n, werr
+		}
+		src = src[nSrc:]
+		if w.n > 0 && len(src) <= n {
+			// Enough bytes from w.src have been consumed. We make src point
+			// to data instead to reduce the copying.
+			w.n = 0
+			n -= len(src)
+			src = data[n:]
+			if n < len(data) && (err == nil || err == ErrShortSrc) {
+				continue
+			}
+		} else {
+			n += nSrc
+		}
+		switch {
+		case err == ErrShortDst && nDst > 0:
+		case err == ErrShortSrc && len(src) < len(w.src):
+			m := copy(w.src, src)
+			// If w.n > 0, bytes from data were already copied to w.src and n
+			// was already set to the number of bytes consumed.
+			if w.n == 0 {
+				n += m
+			}
+			w.n = m
+			return n, nil
+		case err == nil && w.n > 0:
+			return n, errInconsistentByteCount
+		default:
+			return n, err
+		}
+	}
+}
+
+// Close implements the io.Closer interface.
+func (w *Writer) Close() error {
+	for src := w.src[:w.n]; len(src) > 0; {
+		nDst, nSrc, err := w.t.Transform(w.dst, src, true)
+		if nDst == 0 {
+			return err
+		}
+		if _, werr := w.w.Write(w.dst[:nDst]); werr != nil {
+			return werr
+		}
+		if err != ErrShortDst {
+			return err
+		}
+		src = src[nSrc:]
+	}
+	return nil
+}
+
+type nop struct{}
+
+func (nop) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	n := copy(dst, src)
+	if n < len(src) {
+		err = ErrShortDst
+	}
+	return n, n, err
+}
+
+type discard struct{}
+
+func (discard) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	return 0, len(src), nil
+}
+
+var (
+	// Discard is a Transformer for which all Transform calls succeed
+	// by consuming all bytes and writing nothing.
+	Discard Transformer = discard{}
+
+	// Nop is a Transformer that copies src to dst.
+	Nop Transformer = nop{}
+)
+
+// chain is a sequence of links. A chain with N Transformers has N+1 links and
+// N+1 buffers. Of those N+1 buffers, the first and last are the src and dst
+// buffers given to chain.Transform and the middle N-1 buffers are intermediate
+// buffers owned by the chain. The i'th link transforms bytes from the i'th
+// buffer chain.link[i].b at read offset chain.link[i].p to the i+1'th buffer
+// chain.link[i+1].b at write offset chain.link[i+1].n, for i in [0, N).
+type chain struct {
+	link []link
+	err  error
+	// errStart is the index at which the error occurred plus 1. Processing
+	// errStart at this level at the next call to Transform. As long as
+	// errStart > 0, chain will not consume any more source bytes.
+	errStart int
+}
+
+func (c *chain) fatalError(errIndex int, err error) {
+	if i := errIndex + 1; i > c.errStart {
+		c.errStart = i
+		c.err = err
+	}
+}
+
+type link struct {
+	t Transformer
+	// b[p:n] holds the bytes to be transformed by t.
+	b []byte
+	p int
+	n int
+}
+
+func (l *link) src() []byte {
+	return l.b[l.p:l.n]
+}
+
+func (l *link) dst() []byte {
+	return l.b[l.n:]
+}
+
+// Chain returns a Transformer that applies t in sequence.
+func Chain(t ...Transformer) Transformer {
+	if len(t) == 0 {
+		return nop{}
+	}
+	c := &chain{link: make([]link, len(t)+1)}
+	for i, tt := range t {
+		c.link[i].t = tt
+	}
+	// Allocate intermediate buffers.
+	b := make([][defaultBufSize]byte, len(t)-1)
+	for i := range b {
+		c.link[i+1].b = b[i][:]
+	}
+	return c
+}
+
+// Transform applies the transformers of c in sequence.
+func (c *chain) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	// Set up src and dst in the chain.
+	srcL := &c.link[0]
+	dstL := &c.link[len(c.link)-1]
+	srcL.b, srcL.p, srcL.n = src, 0, len(src)
+	dstL.b, dstL.n = dst, 0
+	var lastFull, needProgress bool // for detecting progress
+
+	// i is the index of the next Transformer to apply, for i in [low, high].
+	// low is the lowest index for which c.link[low] may still produce bytes.
+	// high is the highest index for which c.link[high] has a Transformer.
+	// The error returned by Transform determines whether to increase or
+	// decrease i. We try to completely fill a buffer before converting it.
+	for low, i, high := c.errStart, c.errStart, len(c.link)-2; low <= i && i <= high; {
+		in, out := &c.link[i], &c.link[i+1]
+		nDst, nSrc, err0 := in.t.Transform(out.dst(), in.src(), atEOF && low == i)
+		out.n += nDst
+		in.p += nSrc
+		if i > 0 && in.p == in.n {
+			in.p, in.n = 0, 0
+		}
+		needProgress, lastFull = lastFull, false
+		switch err0 {
+		case ErrShortDst:
+			// Process the destination buffer next. Return if we are already
+			// at the high index.
+			if i == high {
+				return dstL.n, srcL.p, ErrShortDst
+			}
+			if out.n != 0 {
+				i++
+				// If the Transformer at the next index is not able to process any
+				// source bytes there is nothing that can be done to make progress
+				// and the bytes will remain unprocessed. lastFull is used to
+				// detect this and break out of the loop with a fatal error.
+				lastFull = true
+				continue
+			}
+			// The destination buffer was too small, but is completely empty.
+			// Return a fatal error as this transformation can never complete.
+			c.fatalError(i, errShortInternal)
+		case ErrShortSrc:
+			if i == 0 {
+				// Save ErrShortSrc in err. All other errors take precedence.
+				err = ErrShortSrc
+				break
+			}
+			// Source bytes were depleted before filling up the destination buffer.
+			// Verify we made some progress, move the remaining bytes to the errStart
+			// and try to get more source bytes.
+			if needProgress && nSrc == 0 || in.n-in.p == len(in.b) {
+				// There were not enough source bytes to proceed while the source
+				// buffer cannot hold any more bytes. Return a fatal error as this
+				// transformation can never complete.
+				c.fatalError(i, errShortInternal)
+				break
+			}
+			// in.b is an internal buffer and we can make progress.
+			in.p, in.n = 0, copy(in.b, in.src())
+			fallthrough
+		case nil:
+			// if i == low, we have depleted the bytes at index i or any lower levels.
+			// In that case we increase low and i. In all other cases we decrease i to
+			// fetch more bytes before proceeding to the next index.
+			if i > low {
+				i--
+				continue
+			}
+		default:
+			c.fatalError(i, err0)
+		}
+		// Exhausted level low or fatal error: increase low and continue
+		// to process the bytes accepted so far.
+		i++
+		low = i
+	}
+
+	// If c.errStart > 0, this means we found a fatal error.  We will clear
+	// all upstream buffers. At this point, no more progress can be made
+	// downstream, as Transform would have bailed while handling ErrShortDst.
+	if c.errStart > 0 {
+		for i := 1; i < c.errStart; i++ {
+			c.link[i].p, c.link[i].n = 0, 0
+		}
+		err, c.errStart, c.err = c.err, 0, nil
+	}
+	return dstL.n, srcL.p, err
+}
+
+// RemoveFunc returns a Transformer that removes from the input all runes r for
+// which f(r) is true. Illegal bytes in the input are replaced by RuneError.
+func RemoveFunc(f func(r rune) bool) Transformer {
+	return removeF(f)
+}
+
+type removeF func(r rune) bool
+
+// Transform implements the Transformer interface.
+func (t removeF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	for r, sz := rune(0), 0; len(src) > 0; src = src[sz:] {
+
+		if r = rune(src[0]); r < utf8.RuneSelf {
+			sz = 1
+		} else {
+			r, sz = utf8.DecodeRune(src)
+
+			if sz == 1 {
+				// Invalid rune.
+				if !atEOF && !utf8.FullRune(src[nSrc:]) {
+					err = ErrShortSrc
+					break
+				}
+				// We replace illegal bytes with RuneError. Not doing so might
+				// otherwise turn a sequence of invalid UTF-8 into valid UTF-8.
+				// The resulting byte sequence may subsequently contain runes
+				// for which t(r) is true that were passed unnoticed.
+				if !t(r) {
+					if nDst+3 > len(dst) {
+						err = ErrShortDst
+						break
+					}
+					nDst += copy(dst[nDst:], "\uFFFD")
+				}
+				nSrc++
+				continue
+			}
+		}
+
+		if !t(r) {
+			if nDst+sz > len(dst) {
+				err = ErrShortDst
+				break
+			}
+			nDst += copy(dst[nDst:], src[:sz])
+		}
+		nSrc += sz
+	}
+	return
+}
+
+// Bytes returns a new byte slice with the result of converting b using t.
+// If any unrecoverable error occurs it returns nil.
+func Bytes(t Transformer, b []byte) []byte {
+	out := make([]byte, len(b))
+	n := 0
+	for {
+		nDst, nSrc, err := t.Transform(out[n:], b, true)
+		n += nDst
+		if err == nil {
+			return out[:n]
+		} else if err != ErrShortDst {
+			return nil
+		}
+		b = b[nSrc:]
+
+		// Grow the destination buffer.
+		sz := len(out)
+		if sz <= 256 {
+			sz *= 2
+		} else {
+			sz += sz >> 1
+		}
+		out2 := make([]byte, sz)
+		copy(out2, out[:n])
+		out = out2
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.text/transform/transform_test.go

@@ -0,0 +1,901 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package transform
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"strconv"
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+type lowerCaseASCII struct{}
+
+func (lowerCaseASCII) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	n := len(src)
+	if n > len(dst) {
+		n, err = len(dst), ErrShortDst
+	}
+	for i, c := range src[:n] {
+		if 'A' <= c && c <= 'Z' {
+			c += 'a' - 'A'
+		}
+		dst[i] = c
+	}
+	return n, n, err
+}
+
+var errYouMentionedX = errors.New("you mentioned X")
+
+type dontMentionX struct{}
+
+func (dontMentionX) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	n := len(src)
+	if n > len(dst) {
+		n, err = len(dst), ErrShortDst
+	}
+	for i, c := range src[:n] {
+		if c == 'X' {
+			return i, i, errYouMentionedX
+		}
+		dst[i] = c
+	}
+	return n, n, err
+}
+
+// doublerAtEOF is a strange Transformer that transforms "this" to "tthhiiss",
+// but only if atEOF is true.
+type doublerAtEOF struct{}
+
+func (doublerAtEOF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	if !atEOF {
+		return 0, 0, ErrShortSrc
+	}
+	for i, c := range src {
+		if 2*i+2 >= len(dst) {
+			return 2 * i, i, ErrShortDst
+		}
+		dst[2*i+0] = c
+		dst[2*i+1] = c
+	}
+	return 2 * len(src), len(src), nil
+}
+
+// rleDecode and rleEncode implement a toy run-length encoding: "aabbbbbbbbbb"
+// is encoded as "2a10b". The decoding is assumed to not contain any numbers.
+
+type rleDecode struct{}
+
+func (rleDecode) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+loop:
+	for len(src) > 0 {
+		n := 0
+		for i, c := range src {
+			if '0' <= c && c <= '9' {
+				n = 10*n + int(c-'0')
+				continue
+			}
+			if i == 0 {
+				return nDst, nSrc, errors.New("rleDecode: bad input")
+			}
+			if n > len(dst) {
+				return nDst, nSrc, ErrShortDst
+			}
+			for j := 0; j < n; j++ {
+				dst[j] = c
+			}
+			dst, src = dst[n:], src[i+1:]
+			nDst, nSrc = nDst+n, nSrc+i+1
+			continue loop
+		}
+		if atEOF {
+			return nDst, nSrc, errors.New("rleDecode: bad input")
+		}
+		return nDst, nSrc, ErrShortSrc
+	}
+	return nDst, nSrc, nil
+}
+
+type rleEncode struct {
+	// allowStutter means that "xxxxxxxx" can be encoded as "5x3x"
+	// instead of always as "8x".
+	allowStutter bool
+}
+
+func (e rleEncode) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	for len(src) > 0 {
+		n, c0 := len(src), src[0]
+		for i, c := range src[1:] {
+			if c != c0 {
+				n = i + 1
+				break
+			}
+		}
+		if n == len(src) && !atEOF && !e.allowStutter {
+			return nDst, nSrc, ErrShortSrc
+		}
+		s := strconv.Itoa(n)
+		if len(s) >= len(dst) {
+			return nDst, nSrc, ErrShortDst
+		}
+		copy(dst, s)
+		dst[len(s)] = c0
+		dst, src = dst[len(s)+1:], src[n:]
+		nDst, nSrc = nDst+len(s)+1, nSrc+n
+	}
+	return nDst, nSrc, nil
+}
+
+type testCase struct {
+	desc     string
+	t        Transformer
+	src      string
+	dstSize  int
+	srcSize  int
+	ioSize   int
+	wantStr  string
+	wantErr  error
+	wantIter int // number of iterations taken; 0 means we don't care.
+}
+
+func (t testCase) String() string {
+	return tstr(t.t) + "; " + t.desc
+}
+
+func tstr(t Transformer) string {
+	if stringer, ok := t.(fmt.Stringer); ok {
+		return stringer.String()
+	}
+	s := fmt.Sprintf("%T", t)
+	return s[1+strings.Index(s, "."):]
+}
+
+func (c chain) String() string {
+	buf := &bytes.Buffer{}
+	buf.WriteString("Chain(")
+	for i, l := range c.link[:len(c.link)-1] {
+		if i != 0 {
+			fmt.Fprint(buf, ", ")
+		}
+		buf.WriteString(tstr(l.t))
+	}
+	buf.WriteString(")")
+	return buf.String()
+}
+
+var testCases = []testCase{
+	{
+		desc:    "basic",
+		t:       lowerCaseASCII{},
+		src:     "Hello WORLD.",
+		dstSize: 100,
+		srcSize: 100,
+		wantStr: "hello world.",
+	},
+
+	{
+		desc:    "small dst",
+		t:       lowerCaseASCII{},
+		src:     "Hello WORLD.",
+		dstSize: 3,
+		srcSize: 100,
+		wantStr: "hello world.",
+	},
+
+	{
+		desc:    "small src",
+		t:       lowerCaseASCII{},
+		src:     "Hello WORLD.",
+		dstSize: 100,
+		srcSize: 4,
+		wantStr: "hello world.",
+	},
+
+	{
+		desc:    "small buffers",
+		t:       lowerCaseASCII{},
+		src:     "Hello WORLD.",
+		dstSize: 3,
+		srcSize: 4,
+		wantStr: "hello world.",
+	},
+
+	{
+		desc:    "very small buffers",
+		t:       lowerCaseASCII{},
+		src:     "Hello WORLD.",
+		dstSize: 1,
+		srcSize: 1,
+		wantStr: "hello world.",
+	},
+
+	{
+		desc:    "basic",
+		t:       dontMentionX{},
+		src:     "The First Rule of Transform Club: don't mention Mister X, ever.",
+		dstSize: 100,
+		srcSize: 100,
+		wantStr: "The First Rule of Transform Club: don't mention Mister ",
+		wantErr: errYouMentionedX,
+	},
+
+	{
+		desc:    "small buffers",
+		t:       dontMentionX{},
+		src:     "The First Rule of Transform Club: don't mention Mister X, ever.",
+		dstSize: 10,
+		srcSize: 10,
+		wantStr: "The First Rule of Transform Club: don't mention Mister ",
+		wantErr: errYouMentionedX,
+	},
+
+	{
+		desc:    "very small buffers",
+		t:       dontMentionX{},
+		src:     "The First Rule of Transform Club: don't mention Mister X, ever.",
+		dstSize: 1,
+		srcSize: 1,
+		wantStr: "The First Rule of Transform Club: don't mention Mister ",
+		wantErr: errYouMentionedX,
+	},
+
+	{
+		desc:    "only transform at EOF",
+		t:       doublerAtEOF{},
+		src:     "this",
+		dstSize: 100,
+		srcSize: 100,
+		wantStr: "tthhiiss",
+	},
+
+	{
+		desc:    "basic",
+		t:       rleDecode{},
+		src:     "1a2b3c10d11e0f1g",
+		dstSize: 100,
+		srcSize: 100,
+		wantStr: "abbcccddddddddddeeeeeeeeeeeg",
+	},
+
+	{
+		desc:    "long",
+		t:       rleDecode{},
+		src:     "12a23b34c45d56e99z",
+		dstSize: 100,
+		srcSize: 100,
+		wantStr: strings.Repeat("a", 12) +
+			strings.Repeat("b", 23) +
+			strings.Repeat("c", 34) +
+			strings.Repeat("d", 45) +
+			strings.Repeat("e", 56) +
+			strings.Repeat("z", 99),
+	},
+
+	{
+		desc:    "tight buffers",
+		t:       rleDecode{},
+		src:     "1a2b3c10d11e0f1g",
+		dstSize: 11,
+		srcSize: 3,
+		wantStr: "abbcccddddddddddeeeeeeeeeeeg",
+	},
+
+	{
+		desc:    "short dst",
+		t:       rleDecode{},
+		src:     "1a2b3c10d11e0f1g",
+		dstSize: 10,
+		srcSize: 3,
+		wantStr: "abbcccdddddddddd",
+		wantErr: ErrShortDst,
+	},
+
+	{
+		desc:    "short src",
+		t:       rleDecode{},
+		src:     "1a2b3c10d11e0f1g",
+		dstSize: 11,
+		srcSize: 2,
+		ioSize:  2,
+		wantStr: "abbccc",
+		wantErr: ErrShortSrc,
+	},
+
+	{
+		desc:    "basic",
+		t:       rleEncode{},
+		src:     "abbcccddddddddddeeeeeeeeeeeg",
+		dstSize: 100,
+		srcSize: 100,
+		wantStr: "1a2b3c10d11e1g",
+	},
+
+	{
+		desc: "long",
+		t:    rleEncode{},
+		src: strings.Repeat("a", 12) +
+			strings.Repeat("b", 23) +
+			strings.Repeat("c", 34) +
+			strings.Repeat("d", 45) +
+			strings.Repeat("e", 56) +
+			strings.Repeat("z", 99),
+		dstSize: 100,
+		srcSize: 100,
+		wantStr: "12a23b34c45d56e99z",
+	},
+
+	{
+		desc:    "tight buffers",
+		t:       rleEncode{},
+		src:     "abbcccddddddddddeeeeeeeeeeeg",
+		dstSize: 3,
+		srcSize: 12,
+		wantStr: "1a2b3c10d11e1g",
+	},
+
+	{
+		desc:    "short dst",
+		t:       rleEncode{},
+		src:     "abbcccddddddddddeeeeeeeeeeeg",
+		dstSize: 2,
+		srcSize: 12,
+		wantStr: "1a2b3c",
+		wantErr: ErrShortDst,
+	},
+
+	{
+		desc:    "short src",
+		t:       rleEncode{},
+		src:     "abbcccddddddddddeeeeeeeeeeeg",
+		dstSize: 3,
+		srcSize: 11,
+		ioSize:  11,
+		wantStr: "1a2b3c10d",
+		wantErr: ErrShortSrc,
+	},
+
+	{
+		desc:    "allowStutter = false",
+		t:       rleEncode{allowStutter: false},
+		src:     "aaaabbbbbbbbccccddddd",
+		dstSize: 10,
+		srcSize: 10,
+		wantStr: "4a8b4c5d",
+	},
+
+	{
+		desc:    "allowStutter = true",
+		t:       rleEncode{allowStutter: true},
+		src:     "aaaabbbbbbbbccccddddd",
+		dstSize: 10,
+		srcSize: 10,
+		ioSize:  10,
+		wantStr: "4a6b2b4c4d1d",
+	},
+}
+
+func TestReader(t *testing.T) {
+	for _, tc := range testCases {
+		reset(tc.t)
+		r := NewReader(strings.NewReader(tc.src), tc.t)
+		// Differently sized dst and src buffers are not part of the
+		// exported API. We override them manually.
+		r.dst = make([]byte, tc.dstSize)
+		r.src = make([]byte, tc.srcSize)
+		got, err := ioutil.ReadAll(r)
+		str := string(got)
+		if str != tc.wantStr || err != tc.wantErr {
+			t.Errorf("%s:\ngot  %q, %v\nwant %q, %v", tc, str, err, tc.wantStr, tc.wantErr)
+		}
+	}
+}
+
+func reset(t Transformer) {
+	var dst [128]byte
+	for err := ErrShortDst; err != nil; {
+		_, _, err = t.Transform(dst[:], nil, true)
+	}
+}
+
+func TestWriter(t *testing.T) {
+	tests := append(testCases, chainTests()...)
+	for _, tc := range tests {
+		sizes := []int{1, 2, 3, 4, 5, 10, 100, 1000}
+		if tc.ioSize > 0 {
+			sizes = []int{tc.ioSize}
+		}
+		for _, sz := range sizes {
+			bb := &bytes.Buffer{}
+			reset(tc.t)
+			w := NewWriter(bb, tc.t)
+			// Differently sized dst and src buffers are not part of the
+			// exported API. We override them manually.
+			w.dst = make([]byte, tc.dstSize)
+			w.src = make([]byte, tc.srcSize)
+			src := make([]byte, sz)
+			var err error
+			for b := tc.src; len(b) > 0 && err == nil; {
+				n := copy(src, b)
+				b = b[n:]
+				m := 0
+				m, err = w.Write(src[:n])
+				if m != n && err == nil {
+					t.Errorf("%s:%d: did not consume all bytes %d < %d", tc, sz, m, n)
+				}
+			}
+			if err == nil {
+				err = w.Close()
+			}
+			str := bb.String()
+			if str != tc.wantStr || err != tc.wantErr {
+				t.Errorf("%s:%d:\ngot  %q, %v\nwant %q, %v", tc, sz, str, err, tc.wantStr, tc.wantErr)
+			}
+		}
+	}
+}
+
+func TestNop(t *testing.T) {
+	testCases := []struct {
+		str     string
+		dstSize int
+		err     error
+	}{
+		{"", 0, nil},
+		{"", 10, nil},
+		{"a", 0, ErrShortDst},
+		{"a", 1, nil},
+		{"a", 10, nil},
+	}
+	for i, tc := range testCases {
+		dst := make([]byte, tc.dstSize)
+		nDst, nSrc, err := Nop.Transform(dst, []byte(tc.str), true)
+		want := tc.str
+		if tc.dstSize < len(want) {
+			want = want[:tc.dstSize]
+		}
+		if got := string(dst[:nDst]); got != want || err != tc.err || nSrc != nDst {
+			t.Errorf("%d:\ngot %q, %d, %v\nwant %q, %d, %v", i, got, nSrc, err, want, nDst, tc.err)
+		}
+	}
+}
+
+func TestDiscard(t *testing.T) {
+	testCases := []struct {
+		str     string
+		dstSize int
+	}{
+		{"", 0},
+		{"", 10},
+		{"a", 0},
+		{"ab", 10},
+	}
+	for i, tc := range testCases {
+		nDst, nSrc, err := Discard.Transform(make([]byte, tc.dstSize), []byte(tc.str), true)
+		if nDst != 0 || nSrc != len(tc.str) || err != nil {
+			t.Errorf("%d:\ngot %q, %d, %v\nwant 0, %d, nil", i, nDst, nSrc, err, len(tc.str))
+		}
+	}
+}
+
+// mkChain creates a Chain transformer. x must be alternating between transformer
+// and bufSize, like T, (sz, T)*
+func mkChain(x ...interface{}) *chain {
+	t := []Transformer{}
+	for i := 0; i < len(x); i += 2 {
+		t = append(t, x[i].(Transformer))
+	}
+	c := Chain(t...).(*chain)
+	for i, j := 1, 1; i < len(x); i, j = i+2, j+1 {
+		c.link[j].b = make([]byte, x[i].(int))
+	}
+	return c
+}
+
+func chainTests() []testCase {
+	return []testCase{
+		{
+			desc:     "nil error",
+			t:        mkChain(rleEncode{}, 100, lowerCaseASCII{}),
+			src:      "ABB",
+			dstSize:  100,
+			srcSize:  100,
+			wantStr:  "1a2b",
+			wantErr:  nil,
+			wantIter: 1,
+		},
+
+		{
+			desc:    "short dst buffer",
+			t:       mkChain(lowerCaseASCII{}, 3, rleDecode{}),
+			src:     "1a2b3c10d11e0f1g",
+			dstSize: 10,
+			srcSize: 3,
+			wantStr: "abbcccdddddddddd",
+			wantErr: ErrShortDst,
+		},
+
+		{
+			desc:    "short internal dst buffer",
+			t:       mkChain(lowerCaseASCII{}, 3, rleDecode{}, 10, Nop),
+			src:     "1a2b3c10d11e0f1g",
+			dstSize: 100,
+			srcSize: 3,
+			wantStr: "abbcccdddddddddd",
+			wantErr: errShortInternal,
+		},
+
+		{
+			desc:    "short internal dst buffer from input",
+			t:       mkChain(rleDecode{}, 10, Nop),
+			src:     "1a2b3c10d11e0f1g",
+			dstSize: 100,
+			srcSize: 3,
+			wantStr: "abbcccdddddddddd",
+			wantErr: errShortInternal,
+		},
+
+		{
+			desc:    "empty short internal dst buffer",
+			t:       mkChain(lowerCaseASCII{}, 3, rleDecode{}, 10, Nop),
+			src:     "4a7b11e0f1g",
+			dstSize: 100,
+			srcSize: 3,
+			wantStr: "aaaabbbbbbb",
+			wantErr: errShortInternal,
+		},
+
+		{
+			desc:    "empty short internal dst buffer from input",
+			t:       mkChain(rleDecode{}, 10, Nop),
+			src:     "4a7b11e0f1g",
+			dstSize: 100,
+			srcSize: 3,
+			wantStr: "aaaabbbbbbb",
+			wantErr: errShortInternal,
+		},
+
+		{
+			desc:     "short internal src buffer after full dst buffer",
+			t:        mkChain(Nop, 5, rleEncode{}, 10, Nop),
+			src:      "cccccddddd",
+			dstSize:  100,
+			srcSize:  100,
+			wantStr:  "",
+			wantErr:  errShortInternal,
+			wantIter: 1,
+		},
+
+		{
+			desc:    "short internal src buffer after short dst buffer; test lastFull",
+			t:       mkChain(rleDecode{}, 5, rleEncode{}, 4, Nop),
+			src:     "2a1b4c6d",
+			dstSize: 100,
+			srcSize: 100,
+			wantStr: "2a1b",
+			wantErr: errShortInternal,
+		},
+
+		{
+			desc:     "short internal src buffer after successful complete fill",
+			t:        mkChain(Nop, 3, rleDecode{}),
+			src:      "123a4b",
+			dstSize:  4,
+			srcSize:  3,
+			wantStr:  "",
+			wantErr:  errShortInternal,
+			wantIter: 1,
+		},
+
+		{
+			desc:    "short internal src buffer after short dst buffer; test lastFull",
+			t:       mkChain(rleDecode{}, 5, rleEncode{}),
+			src:     "2a1b4c6d",
+			dstSize: 4,
+			srcSize: 100,
+			wantStr: "2a1b",
+			wantErr: errShortInternal,
+		},
+
+		{
+			desc:    "short src buffer",
+			t:       mkChain(rleEncode{}, 5, Nop),
+			src:     "abbcccddddeeeee",
+			dstSize: 4,
+			srcSize: 4,
+			ioSize:  4,
+			wantStr: "1a2b3c",
+			wantErr: ErrShortSrc,
+		},
+
+		{
+			desc:     "process all in one go",
+			t:        mkChain(rleEncode{}, 5, Nop),
+			src:      "abbcccddddeeeeeffffff",
+			dstSize:  100,
+			srcSize:  100,
+			wantStr:  "1a2b3c4d5e6f",
+			wantErr:  nil,
+			wantIter: 1,
+		},
+
+		{
+			desc:    "complete processing downstream after error",
+			t:       mkChain(dontMentionX{}, 2, rleDecode{}, 5, Nop),
+			src:     "3a4b5eX",
+			dstSize: 100,
+			srcSize: 100,
+			ioSize:  100,
+			wantStr: "aaabbbbeeeee",
+			wantErr: errYouMentionedX,
+		},
+
+		{
+			desc:    "return downstream fatal errors first (followed by short dst)",
+			t:       mkChain(dontMentionX{}, 8, rleDecode{}, 4, Nop),
+			src:     "3a4b5eX",
+			dstSize: 100,
+			srcSize: 100,
+			ioSize:  100,
+			wantStr: "aaabbbb",
+			wantErr: errShortInternal,
+		},
+
+		{
+			desc:    "return downstream fatal errors first (followed by short src)",
+			t:       mkChain(dontMentionX{}, 5, Nop, 1, rleDecode{}),
+			src:     "1a5bX",
+			dstSize: 100,
+			srcSize: 100,
+			ioSize:  100,
+			wantStr: "",
+			wantErr: errShortInternal,
+		},
+
+		{
+			desc:    "short internal",
+			t:       mkChain(Nop, 11, rleEncode{}, 3, Nop),
+			src:     "abbcccddddddddddeeeeeeeeeeeg",
+			dstSize: 3,
+			srcSize: 100,
+			wantStr: "1a2b3c10d",
+			wantErr: errShortInternal,
+		},
+	}
+}
+
+func doTransform(tc testCase) (res string, iter int, err error) {
+	reset(tc.t)
+	dst := make([]byte, tc.dstSize)
+	out, in := make([]byte, 0, 2*len(tc.src)), []byte(tc.src)
+	for {
+		iter++
+		src, atEOF := in, true
+		if len(src) > tc.srcSize {
+			src, atEOF = src[:tc.srcSize], false
+		}
+		nDst, nSrc, err := tc.t.Transform(dst, src, atEOF)
+		out = append(out, dst[:nDst]...)
+		in = in[nSrc:]
+		switch {
+		case err == nil && len(in) != 0:
+		case err == ErrShortSrc && nSrc > 0:
+		case err == ErrShortDst && nDst > 0:
+		default:
+			return string(out), iter, err
+		}
+	}
+}
+
+func TestChain(t *testing.T) {
+	if c, ok := Chain().(nop); !ok {
+		t.Errorf("empty chain: %v; want Nop", c)
+	}
+
+	// Test Chain for a single Transformer.
+	for _, tc := range testCases {
+		tc.t = Chain(tc.t)
+		str, _, err := doTransform(tc)
+		if str != tc.wantStr || err != tc.wantErr {
+			t.Errorf("%s:\ngot  %q, %v\nwant %q, %v", tc, str, err, tc.wantStr, tc.wantErr)
+		}
+	}
+
+	tests := chainTests()
+	sizes := []int{1, 2, 3, 4, 5, 7, 10, 100, 1000}
+	addTest := func(tc testCase, t *chain) {
+		if t.link[0].t != tc.t && tc.wantErr == ErrShortSrc {
+			tc.wantErr = errShortInternal
+		}
+		if t.link[len(t.link)-2].t != tc.t && tc.wantErr == ErrShortDst {
+			tc.wantErr = errShortInternal
+		}
+		tc.t = t
+		tests = append(tests, tc)
+	}
+	for _, tc := range testCases {
+		for _, sz := range sizes {
+			tt := tc
+			tt.dstSize = sz
+			addTest(tt, mkChain(tc.t, tc.dstSize, Nop))
+			addTest(tt, mkChain(tc.t, tc.dstSize, Nop, 2, Nop))
+			addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, Nop))
+			if sz >= tc.dstSize && (tc.wantErr != ErrShortDst || sz == tc.dstSize) {
+				addTest(tt, mkChain(Nop, tc.srcSize, tc.t))
+				addTest(tt, mkChain(Nop, 100, Nop, tc.srcSize, tc.t))
+			}
+		}
+	}
+	for _, tc := range testCases {
+		tt := tc
+		tt.dstSize = 1
+		tt.wantStr = ""
+		addTest(tt, mkChain(tc.t, tc.dstSize, Discard))
+		addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, Discard))
+		addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, Nop, tc.dstSize, Discard))
+	}
+	for _, tc := range testCases {
+		tt := tc
+		tt.dstSize = 100
+		tt.wantStr = strings.Replace(tc.src, "0f", "", -1)
+		// Chain encoders and decoders.
+		if _, ok := tc.t.(rleEncode); ok && tc.wantErr == nil {
+			addTest(tt, mkChain(tc.t, tc.dstSize, Nop, 1000, rleDecode{}))
+			addTest(tt, mkChain(tc.t, tc.dstSize, Nop, tc.dstSize, rleDecode{}))
+			addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, Nop, 100, rleDecode{}))
+			// decoding needs larger destinations
+			addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, rleDecode{}, 100, Nop))
+			addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, Nop, 100, rleDecode{}, 100, Nop))
+		} else if _, ok := tc.t.(rleDecode); ok && tc.wantErr == nil {
+			// The internal buffer size may need to be the sum of the maximum segment
+			// size of the two encoders!
+			addTest(tt, mkChain(tc.t, 2*tc.dstSize, rleEncode{}))
+			addTest(tt, mkChain(tc.t, tc.dstSize, Nop, 101, rleEncode{}))
+			addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, Nop, 100, rleEncode{}))
+			addTest(tt, mkChain(Nop, tc.srcSize, tc.t, tc.dstSize, Nop, 200, rleEncode{}, 100, Nop))
+		}
+	}
+	for _, tc := range tests {
+		str, iter, err := doTransform(tc)
+		mi := tc.wantIter != 0 && tc.wantIter != iter
+		if str != tc.wantStr || err != tc.wantErr || mi {
+			t.Errorf("%s:\ngot  iter:%d, %q, %v\nwant iter:%d, %q, %v", tc, iter, str, err, tc.wantIter, tc.wantStr, tc.wantErr)
+		}
+		break
+	}
+}
+
+func TestRemoveFunc(t *testing.T) {
+	filter := RemoveFunc(func(r rune) bool {
+		return strings.IndexRune("ab\u0300\u1234,", r) != -1
+	})
+	tests := []testCase{
+		{
+			src:     ",",
+			wantStr: "",
+		},
+
+		{
+			src:     "c",
+			wantStr: "c",
+		},
+
+		{
+			src:     "\u2345",
+			wantStr: "\u2345",
+		},
+
+		{
+			src:     "tschüß",
+			wantStr: "tschüß",
+		},
+
+		{
+			src:     ",до,свидания,",
+			wantStr: "досвидания",
+		},
+
+		{
+			src:     "a\xbd\xb2=\xbc ⌘",
+			wantStr: "\uFFFD\uFFFD=\uFFFD ⌘",
+		},
+
+		{
+			// If we didn't replace illegal bytes with RuneError, the result
+			// would be \u0300 or the code would need to be more complex.
+			src:     "\xcc\u0300\x80",
+			wantStr: "\uFFFD\uFFFD",
+		},
+
+		{
+			src:      "\xcc\u0300\x80",
+			dstSize:  3,
+			wantStr:  "\uFFFD\uFFFD",
+			wantIter: 2,
+		},
+
+		{
+			src:     "\u2345",
+			dstSize: 2,
+			wantStr: "",
+			wantErr: ErrShortDst,
+		},
+
+		{
+			src:     "\xcc",
+			dstSize: 2,
+			wantStr: "",
+			wantErr: ErrShortDst,
+		},
+
+		{
+			src:     "\u0300",
+			dstSize: 2,
+			srcSize: 1,
+			wantStr: "",
+			wantErr: ErrShortSrc,
+		},
+
+		{
+			t: RemoveFunc(func(r rune) bool {
+				return r == utf8.RuneError
+			}),
+			src:     "\xcc\u0300\x80",
+			wantStr: "\u0300",
+		},
+	}
+
+	for _, tc := range tests {
+		tc.desc = tc.src
+		if tc.t == nil {
+			tc.t = filter
+		}
+		if tc.dstSize == 0 {
+			tc.dstSize = 100
+		}
+		if tc.srcSize == 0 {
+			tc.srcSize = 100
+		}
+		str, iter, err := doTransform(tc)
+		mi := tc.wantIter != 0 && tc.wantIter != iter
+		if str != tc.wantStr || err != tc.wantErr || mi {
+			t.Errorf("%+q:\ngot  iter:%d, %+q, %v\nwant iter:%d, %+q, %v", tc.src, iter, str, err, tc.wantIter, tc.wantStr, tc.wantErr)
+		}
+
+		tc.src = str
+		idem, _, _ := doTransform(tc)
+		if str != idem {
+			t.Errorf("%+q: found %+q; want %+q", tc.src, idem, str)
+		}
+	}
+}
+
+func TestBytes(t *testing.T) {
+	for _, tt := range append(testCases, chainTests()...) {
+		if tt.desc == "allowStutter = true" {
+			// We don't have control over the buffer size, so we eliminate tests
+			// that depend on a specific buffer size being set.
+			continue
+		}
+		got := Bytes(tt.t, []byte(tt.src))
+		if tt.wantErr != nil {
+			if tt.wantErr != ErrShortDst && tt.wantErr != ErrShortSrc {
+				// Bytes should return nil for non-recoverable errors.
+				if g, w := (got == nil), (tt.wantErr != nil); g != w {
+					t.Errorf("%s:error: got %v; want %v", tt.desc, g, w)
+				}
+			}
+			// The output strings in the tests that expect an error will
+			// almost certainly not be the same as the result of Bytes.
+			continue
+		}
+		if string(got) != tt.wantStr {
+			t.Errorf("%s:string: got %q; want %q", tt.desc, got, tt.wantStr)
+		}
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/Makefile

@@ -0,0 +1,30 @@
+# Copyright 2011 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+maketables: maketables.go triegen.go
+	go build $^
+
+maketesttables: maketesttables.go triegen.go
+	go build $^
+
+normregtest: normregtest.go
+	go build $^
+
+tables:	maketables
+	./maketables > tables.go
+	gofmt -w tables.go
+
+trietesttables: maketesttables
+	./maketesttables > triedata_test.go
+	gofmt -w triedata_test.go
+
+# Downloads from www.unicode.org, so not part
+# of standard test scripts.
+test: testtables regtest
+
+testtables: maketables
+	./maketables -test > data_test.go && go test -tags=test
+
+regtest: normregtest
+	./normregtest

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/composition.go

@@ -0,0 +1,514 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "unicode/utf8"
+
+const (
+	maxNonStarters = 30
+	// The maximum number of characters needed for a buffer is
+	// maxNonStarters + 1 for the starter + 1 for the GCJ
+	maxBufferSize    = maxNonStarters + 2
+	maxNFCExpansion  = 3  // NFC(0x1D160)
+	maxNFKCExpansion = 18 // NFKC(0xFDFA)
+
+	maxByteBufferSize = utf8.UTFMax * maxBufferSize // 128
+)
+
+// ssState is used for reporting the segment state after inserting a rune.
+// It is returned by streamSafe.next.
+type ssState int
+
+const (
+	// Indicates a rune was successfully added to the segment.
+	ssSuccess ssState = iota
+	// Indicates a rune starts a new segment and should not be added.
+	ssStarter
+	// Indicates a rune caused a segment overflow and a CGJ should be inserted.
+	ssOverflow
+)
+
+// streamSafe implements the policy of when a CGJ should be inserted.
+type streamSafe uint8
+
+// mkStreamSafe is a shorthand for declaring a streamSafe var and calling
+// first on it.
+func mkStreamSafe(p Properties) streamSafe {
+	return streamSafe(p.nTrailingNonStarters())
+}
+
+// first inserts the first rune of a segment.
+func (ss *streamSafe) first(p Properties) {
+	if *ss != 0 {
+		panic("!= 0")
+	}
+	*ss = streamSafe(p.nTrailingNonStarters())
+}
+
+// insert returns a ssState value to indicate whether a rune represented by p
+// can be inserted.
+func (ss *streamSafe) next(p Properties) ssState {
+	if *ss > maxNonStarters {
+		panic("streamSafe was not reset")
+	}
+	n := p.nLeadingNonStarters()
+	if *ss += streamSafe(n); *ss > maxNonStarters {
+		*ss = 0
+		return ssOverflow
+	}
+	// The Stream-Safe Text Processing prescribes that the counting can stop
+	// as soon as a starter is encountered. However, there are some starters,
+	// like Jamo V and T, that can combine with other runes, leaving their
+	// successive non-starters appended to the previous, possibly causing an
+	// overflow. We will therefore consider any rune with a non-zero nLead to
+	// be a non-starter. Note that it always hold that if nLead > 0 then
+	// nLead == nTrail.
+	if n == 0 {
+		*ss = 0
+		return ssStarter
+	}
+	return ssSuccess
+}
+
+// backwards is used for checking for overflow and segment starts
+// when traversing a string backwards. Users do not need to call first
+// for the first rune. The state of the streamSafe retains the count of
+// the non-starters loaded.
+func (ss *streamSafe) backwards(p Properties) ssState {
+	if *ss > maxNonStarters {
+		panic("streamSafe was not reset")
+	}
+	c := *ss + streamSafe(p.nTrailingNonStarters())
+	if c > maxNonStarters {
+		return ssOverflow
+	}
+	*ss = c
+	if p.nLeadingNonStarters() == 0 {
+		return ssStarter
+	}
+	return ssSuccess
+}
+
+func (ss streamSafe) isMax() bool {
+	return ss == maxNonStarters
+}
+
+// GraphemeJoiner is inserted after maxNonStarters non-starter runes.
+const GraphemeJoiner = "\u034F"
+
+// reorderBuffer is used to normalize a single segment.  Characters inserted with
+// insert are decomposed and reordered based on CCC. The compose method can
+// be used to recombine characters.  Note that the byte buffer does not hold
+// the UTF-8 characters in order.  Only the rune array is maintained in sorted
+// order. flush writes the resulting segment to a byte array.
+type reorderBuffer struct {
+	rune  [maxBufferSize]Properties // Per character info.
+	byte  [maxByteBufferSize]byte   // UTF-8 buffer. Referenced by runeInfo.pos.
+	nbyte uint8                     // Number or bytes.
+	ss    streamSafe                // For limiting length of non-starter sequence.
+	nrune int                       // Number of runeInfos.
+	f     formInfo
+
+	src      input
+	nsrc     int
+	tmpBytes input
+
+	out    []byte
+	flushF func(*reorderBuffer) bool
+}
+
+func (rb *reorderBuffer) init(f Form, src []byte) {
+	rb.f = *formTable[f]
+	rb.src.setBytes(src)
+	rb.nsrc = len(src)
+	rb.ss = 0
+}
+
+func (rb *reorderBuffer) initString(f Form, src string) {
+	rb.f = *formTable[f]
+	rb.src.setString(src)
+	rb.nsrc = len(src)
+	rb.ss = 0
+}
+
+func (rb *reorderBuffer) setFlusher(out []byte, f func(*reorderBuffer) bool) {
+	rb.out = out
+	rb.flushF = f
+}
+
+// reset discards all characters from the buffer.
+func (rb *reorderBuffer) reset() {
+	rb.nrune = 0
+	rb.nbyte = 0
+	rb.ss = 0
+}
+
+func (rb *reorderBuffer) doFlush() bool {
+	if rb.f.composing {
+		rb.compose()
+	}
+	res := rb.flushF(rb)
+	rb.reset()
+	return res
+}
+
+// appendFlush appends the normalized segment to rb.out.
+func appendFlush(rb *reorderBuffer) bool {
+	for i := 0; i < rb.nrune; i++ {
+		start := rb.rune[i].pos
+		end := start + rb.rune[i].size
+		rb.out = append(rb.out, rb.byte[start:end]...)
+	}
+	return true
+}
+
+// flush appends the normalized segment to out and resets rb.
+func (rb *reorderBuffer) flush(out []byte) []byte {
+	for i := 0; i < rb.nrune; i++ {
+		start := rb.rune[i].pos
+		end := start + rb.rune[i].size
+		out = append(out, rb.byte[start:end]...)
+	}
+	rb.reset()
+	return out
+}
+
+// flushCopy copies the normalized segment to buf and resets rb.
+// It returns the number of bytes written to buf.
+func (rb *reorderBuffer) flushCopy(buf []byte) int {
+	p := 0
+	for i := 0; i < rb.nrune; i++ {
+		runep := rb.rune[i]
+		p += copy(buf[p:], rb.byte[runep.pos:runep.pos+runep.size])
+	}
+	rb.reset()
+	return p
+}
+
+// insertOrdered inserts a rune in the buffer, ordered by Canonical Combining Class.
+// It returns false if the buffer is not large enough to hold the rune.
+// It is used internally by insert and insertString only.
+func (rb *reorderBuffer) insertOrdered(info Properties) {
+	n := rb.nrune
+	b := rb.rune[:]
+	cc := info.ccc
+	if cc > 0 {
+		// Find insertion position + move elements to make room.
+		for ; n > 0; n-- {
+			if b[n-1].ccc <= cc {
+				break
+			}
+			b[n] = b[n-1]
+		}
+	}
+	rb.nrune += 1
+	pos := uint8(rb.nbyte)
+	rb.nbyte += utf8.UTFMax
+	info.pos = pos
+	b[n] = info
+}
+
+// insertErr is an error code returned by insert. Using this type instead
+// of error improves performance up to 20% for many of the benchmarks.
+type insertErr int
+
+const (
+	iSuccess insertErr = -iota
+	iShortDst
+	iShortSrc
+)
+
+// insertFlush inserts the given rune in the buffer ordered by CCC.
+// If a decomposition with multiple segments are encountered, they leading
+// ones are flushed.
+// It returns a non-zero error code if the rune was not inserted.
+func (rb *reorderBuffer) insertFlush(src input, i int, info Properties) insertErr {
+	if rune := src.hangul(i); rune != 0 {
+		rb.decomposeHangul(rune)
+		return iSuccess
+	}
+	if info.hasDecomposition() {
+		return rb.insertDecomposed(info.Decomposition())
+	}
+	rb.insertSingle(src, i, info)
+	return iSuccess
+}
+
+// insertUnsafe inserts the given rune in the buffer ordered by CCC.
+// It is assumed there is sufficient space to hold the runes. It is the
+// responsibility of the caller to ensure this. This can be done by checking
+// the state returned by the streamSafe type.
+func (rb *reorderBuffer) insertUnsafe(src input, i int, info Properties) {
+	if rune := src.hangul(i); rune != 0 {
+		rb.decomposeHangul(rune)
+	}
+	if info.hasDecomposition() {
+		// TODO: inline.
+		rb.insertDecomposed(info.Decomposition())
+	} else {
+		rb.insertSingle(src, i, info)
+	}
+}
+
+// insertDecomposed inserts an entry in to the reorderBuffer for each rune
+// in dcomp. dcomp must be a sequence of decomposed UTF-8-encoded runes.
+// It flushes the buffer on each new segment start.
+func (rb *reorderBuffer) insertDecomposed(dcomp []byte) insertErr {
+	rb.tmpBytes.setBytes(dcomp)
+	for i := 0; i < len(dcomp); {
+		info := rb.f.info(rb.tmpBytes, i)
+		if info.BoundaryBefore() && rb.nrune > 0 && !rb.doFlush() {
+			return iShortDst
+		}
+		i += copy(rb.byte[rb.nbyte:], dcomp[i:i+int(info.size)])
+		rb.insertOrdered(info)
+	}
+	return iSuccess
+}
+
+// insertSingle inserts an entry in the reorderBuffer for the rune at
+// position i. info is the runeInfo for the rune at position i.
+func (rb *reorderBuffer) insertSingle(src input, i int, info Properties) {
+	src.copySlice(rb.byte[rb.nbyte:], i, i+int(info.size))
+	rb.insertOrdered(info)
+}
+
+// insertCGJ inserts a Combining Grapheme Joiner (0x034f) into rb.
+func (rb *reorderBuffer) insertCGJ() {
+	rb.insertSingle(input{str: GraphemeJoiner}, 0, Properties{size: uint8(len(GraphemeJoiner))})
+}
+
+// appendRune inserts a rune at the end of the buffer. It is used for Hangul.
+func (rb *reorderBuffer) appendRune(r rune) {
+	bn := rb.nbyte
+	sz := utf8.EncodeRune(rb.byte[bn:], rune(r))
+	rb.nbyte += utf8.UTFMax
+	rb.rune[rb.nrune] = Properties{pos: bn, size: uint8(sz)}
+	rb.nrune++
+}
+
+// assignRune sets a rune at position pos. It is used for Hangul and recomposition.
+func (rb *reorderBuffer) assignRune(pos int, r rune) {
+	bn := rb.rune[pos].pos
+	sz := utf8.EncodeRune(rb.byte[bn:], rune(r))
+	rb.rune[pos] = Properties{pos: bn, size: uint8(sz)}
+}
+
+// runeAt returns the rune at position n. It is used for Hangul and recomposition.
+func (rb *reorderBuffer) runeAt(n int) rune {
+	inf := rb.rune[n]
+	r, _ := utf8.DecodeRune(rb.byte[inf.pos : inf.pos+inf.size])
+	return r
+}
+
+// bytesAt returns the UTF-8 encoding of the rune at position n.
+// It is used for Hangul and recomposition.
+func (rb *reorderBuffer) bytesAt(n int) []byte {
+	inf := rb.rune[n]
+	return rb.byte[inf.pos : int(inf.pos)+int(inf.size)]
+}
+
+// For Hangul we combine algorithmically, instead of using tables.
+const (
+	hangulBase  = 0xAC00 // UTF-8(hangulBase) -> EA B0 80
+	hangulBase0 = 0xEA
+	hangulBase1 = 0xB0
+	hangulBase2 = 0x80
+
+	hangulEnd  = hangulBase + jamoLVTCount // UTF-8(0xD7A4) -> ED 9E A4
+	hangulEnd0 = 0xED
+	hangulEnd1 = 0x9E
+	hangulEnd2 = 0xA4
+
+	jamoLBase  = 0x1100 // UTF-8(jamoLBase) -> E1 84 00
+	jamoLBase0 = 0xE1
+	jamoLBase1 = 0x84
+	jamoLEnd   = 0x1113
+	jamoVBase  = 0x1161
+	jamoVEnd   = 0x1176
+	jamoTBase  = 0x11A7
+	jamoTEnd   = 0x11C3
+
+	jamoTCount   = 28
+	jamoVCount   = 21
+	jamoVTCount  = 21 * 28
+	jamoLVTCount = 19 * 21 * 28
+)
+
+const hangulUTF8Size = 3
+
+func isHangul(b []byte) bool {
+	if len(b) < hangulUTF8Size {
+		return false
+	}
+	b0 := b[0]
+	if b0 < hangulBase0 {
+		return false
+	}
+	b1 := b[1]
+	switch {
+	case b0 == hangulBase0:
+		return b1 >= hangulBase1
+	case b0 < hangulEnd0:
+		return true
+	case b0 > hangulEnd0:
+		return false
+	case b1 < hangulEnd1:
+		return true
+	}
+	return b1 == hangulEnd1 && b[2] < hangulEnd2
+}
+
+func isHangulString(b string) bool {
+	if len(b) < hangulUTF8Size {
+		return false
+	}
+	b0 := b[0]
+	if b0 < hangulBase0 {
+		return false
+	}
+	b1 := b[1]
+	switch {
+	case b0 == hangulBase0:
+		return b1 >= hangulBase1
+	case b0 < hangulEnd0:
+		return true
+	case b0 > hangulEnd0:
+		return false
+	case b1 < hangulEnd1:
+		return true
+	}
+	return b1 == hangulEnd1 && b[2] < hangulEnd2
+}
+
+// Caller must ensure len(b) >= 2.
+func isJamoVT(b []byte) bool {
+	// True if (rune & 0xff00) == jamoLBase
+	return b[0] == jamoLBase0 && (b[1]&0xFC) == jamoLBase1
+}
+
+func isHangulWithoutJamoT(b []byte) bool {
+	c, _ := utf8.DecodeRune(b)
+	c -= hangulBase
+	return c < jamoLVTCount && c%jamoTCount == 0
+}
+
+// decomposeHangul writes the decomposed Hangul to buf and returns the number
+// of bytes written.  len(buf) should be at least 9.
+func decomposeHangul(buf []byte, r rune) int {
+	const JamoUTF8Len = 3
+	r -= hangulBase
+	x := r % jamoTCount
+	r /= jamoTCount
+	utf8.EncodeRune(buf, jamoLBase+r/jamoVCount)
+	utf8.EncodeRune(buf[JamoUTF8Len:], jamoVBase+r%jamoVCount)
+	if x != 0 {
+		utf8.EncodeRune(buf[2*JamoUTF8Len:], jamoTBase+x)
+		return 3 * JamoUTF8Len
+	}
+	return 2 * JamoUTF8Len
+}
+
+// decomposeHangul algorithmically decomposes a Hangul rune into
+// its Jamo components.
+// See http://unicode.org/reports/tr15/#Hangul for details on decomposing Hangul.
+func (rb *reorderBuffer) decomposeHangul(r rune) {
+	r -= hangulBase
+	x := r % jamoTCount
+	r /= jamoTCount
+	rb.appendRune(jamoLBase + r/jamoVCount)
+	rb.appendRune(jamoVBase + r%jamoVCount)
+	if x != 0 {
+		rb.appendRune(jamoTBase + x)
+	}
+}
+
+// combineHangul algorithmically combines Jamo character components into Hangul.
+// See http://unicode.org/reports/tr15/#Hangul for details on combining Hangul.
+func (rb *reorderBuffer) combineHangul(s, i, k int) {
+	b := rb.rune[:]
+	bn := rb.nrune
+	for ; i < bn; i++ {
+		cccB := b[k-1].ccc
+		cccC := b[i].ccc
+		if cccB == 0 {
+			s = k - 1
+		}
+		if s != k-1 && cccB >= cccC {
+			// b[i] is blocked by greater-equal cccX below it
+			b[k] = b[i]
+			k++
+		} else {
+			l := rb.runeAt(s) // also used to compare to hangulBase
+			v := rb.runeAt(i) // also used to compare to jamoT
+			switch {
+			case jamoLBase <= l && l < jamoLEnd &&
+				jamoVBase <= v && v < jamoVEnd:
+				// 11xx plus 116x to LV
+				rb.assignRune(s, hangulBase+
+					(l-jamoLBase)*jamoVTCount+(v-jamoVBase)*jamoTCount)
+			case hangulBase <= l && l < hangulEnd &&
+				jamoTBase < v && v < jamoTEnd &&
+				((l-hangulBase)%jamoTCount) == 0:
+				// ACxx plus 11Ax to LVT
+				rb.assignRune(s, l+v-jamoTBase)
+			default:
+				b[k] = b[i]
+				k++
+			}
+		}
+	}
+	rb.nrune = k
+}
+
+// compose recombines the runes in the buffer.
+// It should only be used to recompose a single segment, as it will not
+// handle alternations between Hangul and non-Hangul characters correctly.
+func (rb *reorderBuffer) compose() {
+	// UAX #15, section X5 , including Corrigendum #5
+	// "In any character sequence beginning with starter S, a character C is
+	//  blocked from S if and only if there is some character B between S
+	//  and C, and either B is a starter or it has the same or higher
+	//  combining class as C."
+	bn := rb.nrune
+	if bn == 0 {
+		return
+	}
+	k := 1
+	b := rb.rune[:]
+	for s, i := 0, 1; i < bn; i++ {
+		if isJamoVT(rb.bytesAt(i)) {
+			// Redo from start in Hangul mode. Necessary to support
+			// U+320E..U+321E in NFKC mode.
+			rb.combineHangul(s, i, k)
+			return
+		}
+		ii := b[i]
+		// We can only use combineForward as a filter if we later
+		// get the info for the combined character. This is more
+		// expensive than using the filter. Using combinesBackward()
+		// is safe.
+		if ii.combinesBackward() {
+			cccB := b[k-1].ccc
+			cccC := ii.ccc
+			blocked := false // b[i] blocked by starter or greater or equal CCC?
+			if cccB == 0 {
+				s = k - 1
+			} else {
+				blocked = s != k-1 && cccB >= cccC
+			}
+			if !blocked {
+				combined := combine(rb.runeAt(s), rb.runeAt(i))
+				if combined != 0 {
+					rb.assignRune(s, combined)
+					continue
+				}
+			}
+		}
+		b[k] = b[i]
+		k++
+	}
+	rb.nrune = k
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/composition_test.go

@@ -0,0 +1,130 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "testing"
+
+// TestCase is used for most tests.
+type TestCase struct {
+	in  []rune
+	out []rune
+}
+
+func runTests(t *testing.T, name string, fm Form, tests []TestCase) {
+	rb := reorderBuffer{}
+	rb.init(fm, nil)
+	for i, test := range tests {
+		rb.setFlusher(nil, appendFlush)
+		for j, rune := range test.in {
+			b := []byte(string(rune))
+			src := inputBytes(b)
+			info := rb.f.info(src, 0)
+			if j == 0 {
+				rb.ss.first(info)
+			} else {
+				rb.ss.next(info)
+			}
+			if rb.insertFlush(src, 0, info) < 0 {
+				t.Errorf("%s:%d: insert failed for rune %d", name, i, j)
+			}
+		}
+		rb.doFlush()
+		was := string(rb.out)
+		want := string(test.out)
+		if len(was) != len(want) {
+			t.Errorf("%s:%d: length = %d; want %d", name, i, len(was), len(want))
+		}
+		if was != want {
+			k, pfx := pidx(was, want)
+			t.Errorf("%s:%d: \nwas  %s%+q; \nwant %s%+q", name, i, pfx, was[k:], pfx, want[k:])
+		}
+	}
+}
+
+func TestFlush(t *testing.T) {
+	const (
+		hello = "Hello "
+		world = "world!"
+	)
+	buf := make([]byte, maxByteBufferSize)
+	p := copy(buf, hello)
+	out := buf[p:]
+	rb := reorderBuffer{}
+	rb.initString(NFC, world)
+	if i := rb.flushCopy(out); i != 0 {
+		t.Errorf("wrote bytes on flush of empty buffer. (len(out) = %d)", i)
+	}
+
+	for i := range world {
+		// No need to set streamSafe values for this test.
+		rb.insertFlush(rb.src, i, rb.f.info(rb.src, i))
+		n := rb.flushCopy(out)
+		out = out[n:]
+		p += n
+	}
+
+	was := buf[:p]
+	want := hello + world
+	if string(was) != want {
+		t.Errorf(`output after flush was "%s"; want "%s"`, string(was), want)
+	}
+	if rb.nrune != 0 {
+		t.Errorf("non-null size of info buffer (rb.nrune == %d)", rb.nrune)
+	}
+	if rb.nbyte != 0 {
+		t.Errorf("non-null size of byte buffer (rb.nbyte == %d)", rb.nbyte)
+	}
+}
+
+var insertTests = []TestCase{
+	{[]rune{'a'}, []rune{'a'}},
+	{[]rune{0x300}, []rune{0x300}},
+	{[]rune{0x300, 0x316}, []rune{0x316, 0x300}}, // CCC(0x300)==230; CCC(0x316)==220
+	{[]rune{0x316, 0x300}, []rune{0x316, 0x300}},
+	{[]rune{0x41, 0x316, 0x300}, []rune{0x41, 0x316, 0x300}},
+	{[]rune{0x41, 0x300, 0x316}, []rune{0x41, 0x316, 0x300}},
+	{[]rune{0x300, 0x316, 0x41}, []rune{0x316, 0x300, 0x41}},
+	{[]rune{0x41, 0x300, 0x40, 0x316}, []rune{0x41, 0x300, 0x40, 0x316}},
+}
+
+func TestInsert(t *testing.T) {
+	runTests(t, "TestInsert", NFD, insertTests)
+}
+
+var decompositionNFDTest = []TestCase{
+	{[]rune{0xC0}, []rune{0x41, 0x300}},
+	{[]rune{0xAC00}, []rune{0x1100, 0x1161}},
+	{[]rune{0x01C4}, []rune{0x01C4}},
+	{[]rune{0x320E}, []rune{0x320E}},
+	{[]rune("음ẻ과"), []rune{0x110B, 0x1173, 0x11B7, 0x65, 0x309, 0x1100, 0x116A}},
+}
+
+var decompositionNFKDTest = []TestCase{
+	{[]rune{0xC0}, []rune{0x41, 0x300}},
+	{[]rune{0xAC00}, []rune{0x1100, 0x1161}},
+	{[]rune{0x01C4}, []rune{0x44, 0x5A, 0x030C}},
+	{[]rune{0x320E}, []rune{0x28, 0x1100, 0x1161, 0x29}},
+}
+
+func TestDecomposition(t *testing.T) {
+	runTests(t, "TestDecompositionNFD", NFD, decompositionNFDTest)
+	runTests(t, "TestDecompositionNFKD", NFKD, decompositionNFKDTest)
+}
+
+var compositionTest = []TestCase{
+	{[]rune{0x41, 0x300}, []rune{0xC0}},
+	{[]rune{0x41, 0x316}, []rune{0x41, 0x316}},
+	{[]rune{0x41, 0x300, 0x35D}, []rune{0xC0, 0x35D}},
+	{[]rune{0x41, 0x316, 0x300}, []rune{0xC0, 0x316}},
+	// blocking starter
+	{[]rune{0x41, 0x316, 0x40, 0x300}, []rune{0x41, 0x316, 0x40, 0x300}},
+	{[]rune{0x1100, 0x1161}, []rune{0xAC00}},
+	// parenthesized Hangul, alternate between ASCII and Hangul.
+	{[]rune{0x28, 0x1100, 0x1161, 0x29}, []rune{0x28, 0xAC00, 0x29}},
+}
+
+func TestComposition(t *testing.T) {
+	runTests(t, "TestComposition", NFC, compositionTest)
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/example_iter_test.go

@@ -0,0 +1,82 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm_test
+
+import (
+	"bytes"
+	"fmt"
+	"unicode/utf8"
+
+	"code.google.com/p/go.text/unicode/norm"
+)
+
+// EqualSimple uses a norm.Iter to compare two non-normalized
+// strings for equivalence.
+func EqualSimple(a, b string) bool {
+	var ia, ib norm.Iter
+	ia.InitString(norm.NFKD, a)
+	ib.InitString(norm.NFKD, b)
+	for !ia.Done() && !ib.Done() {
+		if !bytes.Equal(ia.Next(), ib.Next()) {
+			return false
+		}
+	}
+	return ia.Done() && ib.Done()
+}
+
+// FindPrefix finds the longest common prefix of ASCII characters
+// of a and b.
+func FindPrefix(a, b string) int {
+	i := 0
+	for ; i < len(a) && i < len(b) && a[i] < utf8.RuneSelf && a[i] == b[i]; i++ {
+	}
+	return i
+}
+
+// EqualOpt is like EqualSimple, but optimizes the special
+// case for ASCII characters.
+func EqualOpt(a, b string) bool {
+	n := FindPrefix(a, b)
+	a, b = a[n:], b[n:]
+	var ia, ib norm.Iter
+	ia.InitString(norm.NFKD, a)
+	ib.InitString(norm.NFKD, b)
+	for !ia.Done() && !ib.Done() {
+		if !bytes.Equal(ia.Next(), ib.Next()) {
+			return false
+		}
+		if n := int64(FindPrefix(a[ia.Pos():], b[ib.Pos():])); n != 0 {
+			ia.Seek(n, 1)
+			ib.Seek(n, 1)
+		}
+	}
+	return ia.Done() && ib.Done()
+}
+
+var compareTests = []struct{ a, b string }{
+	{"aaa", "aaa"},
+	{"aaa", "aab"},
+	{"a\u0300a", "\u00E0a"},
+	{"a\u0300\u0320b", "a\u0320\u0300b"},
+	{"\u1E0A\u0323", "\x44\u0323\u0307"},
+	// A character that decomposes into multiple segments
+	// spans several iterations.
+	{"\u3304", "\u30A4\u30CB\u30F3\u30AF\u3099"},
+}
+
+func ExampleIter() {
+	for i, t := range compareTests {
+		r0 := EqualSimple(t.a, t.b)
+		r1 := EqualOpt(t.a, t.b)
+		fmt.Printf("%d: %v %v\n", i, r0, r1)
+	}
+	// Output:
+	// 0: true true
+	// 1: false false
+	// 2: true true
+	// 3: true true
+	// 4: true true
+	// 5: true true
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/forminfo.go

@@ -0,0 +1,256 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+// This file contains Form-specific logic and wrappers for data in tables.go.
+
+// Rune info is stored in a separate trie per composing form. A composing form
+// and its corresponding decomposing form share the same trie.  Each trie maps
+// a rune to a uint16. The values take two forms.  For v >= 0x8000:
+//   bits
+//   15:    1 (inverse of NFD_QD bit of qcInfo)
+//   13..7: qcInfo (see below). isYesD is always true (no decompostion).
+//    6..0: ccc (compressed CCC value).
+// For v < 0x8000, the respective rune has a decomposition and v is an index
+// into a byte array of UTF-8 decomposition sequences and additional info and
+// has the form:
+//    <header> <decomp_byte>* [<tccc> [<lccc>]]
+// The header contains the number of bytes in the decomposition (excluding this
+// length byte). The two most significant bits of this length byte correspond
+// to bit 5 and 4 of qcInfo (see below).  The byte sequence itself starts at v+1.
+// The byte sequence is followed by a trailing and leading CCC if the values
+// for these are not zero.  The value of v determines which ccc are appended
+// to the sequences.  For v < firstCCC, there are none, for v >= firstCCC,
+// the sequence is followed by a trailing ccc, and for v >= firstLeadingCC
+// there is an additional leading ccc. The value of tccc itself is the
+// trailing CCC shifted left 2 bits. The two least-significant bits of tccc
+// are the number of trailing non-starters.
+
+const (
+	qcInfoMask      = 0x3F // to clear all but the relevant bits in a qcInfo
+	headerLenMask   = 0x3F // extract the length value from the header byte
+	headerFlagsMask = 0xC0 // extract the qcInfo bits from the header byte
+)
+
+// Properties provides access to normalization properties of a rune.
+type Properties struct {
+	pos   uint8  // start position in reorderBuffer; used in composition.go
+	size  uint8  // length of UTF-8 encoding of this rune
+	ccc   uint8  // leading canonical combining class (ccc if not decomposition)
+	tccc  uint8  // trailing canonical combining class (ccc if not decomposition)
+	nLead uint8  // number of leading non-starters.
+	flags qcInfo // quick check flags
+	index uint16
+}
+
+// functions dispatchable per form
+type lookupFunc func(b input, i int) Properties
+
+// formInfo holds Form-specific functions and tables.
+type formInfo struct {
+	form                     Form
+	composing, compatibility bool // form type
+	info                     lookupFunc
+	nextMain                 iterFunc
+}
+
+var formTable []*formInfo
+
+func init() {
+	formTable = make([]*formInfo, 4)
+
+	for i := range formTable {
+		f := &formInfo{}
+		formTable[i] = f
+		f.form = Form(i)
+		if Form(i) == NFKD || Form(i) == NFKC {
+			f.compatibility = true
+			f.info = lookupInfoNFKC
+		} else {
+			f.info = lookupInfoNFC
+		}
+		f.nextMain = nextDecomposed
+		if Form(i) == NFC || Form(i) == NFKC {
+			f.nextMain = nextComposed
+			f.composing = true
+		}
+	}
+}
+
+// We do not distinguish between boundaries for NFC, NFD, etc. to avoid
+// unexpected behavior for the user.  For example, in NFD, there is a boundary
+// after 'a'.  However, 'a' might combine with modifiers, so from the application's
+// perspective it is not a good boundary. We will therefore always use the
+// boundaries for the combining variants.
+
+// BoundaryBefore returns true if this rune starts a new segment and
+// cannot combine with any rune on the left.
+func (p Properties) BoundaryBefore() bool {
+	if p.ccc == 0 && !p.combinesBackward() {
+		return true
+	}
+	// We assume that the CCC of the first character in a decomposition
+	// is always non-zero if different from info.ccc and that we can return
+	// false at this point. This is verified by maketables.
+	return false
+}
+
+// BoundaryAfter returns true if runes cannot combine with or otherwise
+// interact with this or previous runes.
+func (p Properties) BoundaryAfter() bool {
+	// TODO: loosen these conditions.
+	return p.isInert()
+}
+
+// We pack quick check data in 4 bits:
+//   5:    Combines forward  (0 == false, 1 == true)
+//   4..3: NFC_QC Yes(00), No (10), or Maybe (11)
+//   2:    NFD_QC Yes (0) or No (1). No also means there is a decomposition.
+//   1..0: Number of trailing non-starters.
+//
+// When all 4 bits are zero, the character is inert, meaning it is never
+// influenced by normalization.
+type qcInfo uint8
+
+func (p Properties) isYesC() bool { return p.flags&0x10 == 0 }
+func (p Properties) isYesD() bool { return p.flags&0x4 == 0 }
+
+func (p Properties) combinesForward() bool  { return p.flags&0x20 != 0 }
+func (p Properties) combinesBackward() bool { return p.flags&0x8 != 0 } // == isMaybe
+func (p Properties) hasDecomposition() bool { return p.flags&0x4 != 0 } // == isNoD
+
+func (p Properties) isInert() bool {
+	return p.flags&qcInfoMask == 0 && p.ccc == 0
+}
+
+func (p Properties) multiSegment() bool {
+	return p.index >= firstMulti && p.index < endMulti
+}
+
+func (p Properties) nLeadingNonStarters() uint8 {
+	return p.nLead
+}
+
+func (p Properties) nTrailingNonStarters() uint8 {
+	return uint8(p.flags & 0x03)
+}
+
+// Decomposition returns the decomposition for the underlying rune
+// or nil if there is none.
+func (p Properties) Decomposition() []byte {
+	// TODO: create the decomposition for Hangul?
+	if p.index == 0 {
+		return nil
+	}
+	i := p.index
+	n := decomps[i] & headerLenMask
+	i++
+	return decomps[i : i+uint16(n)]
+}
+
+// Size returns the length of UTF-8 encoding of the rune.
+func (p Properties) Size() int {
+	return int(p.size)
+}
+
+// CCC returns the canonical combining class of the underlying rune.
+func (p Properties) CCC() uint8 {
+	if p.index >= firstCCCZeroExcept {
+		return 0
+	}
+	return ccc[p.ccc]
+}
+
+// LeadCCC returns the CCC of the first rune in the decomposition.
+// If there is no decomposition, LeadCCC equals CCC.
+func (p Properties) LeadCCC() uint8 {
+	return ccc[p.ccc]
+}
+
+// TrailCCC returns the CCC of the last rune in the decomposition.
+// If there is no decomposition, TrailCCC equals CCC.
+func (p Properties) TrailCCC() uint8 {
+	return ccc[p.tccc]
+}
+
+// Recomposition
+// We use 32-bit keys instead of 64-bit for the two codepoint keys.
+// This clips off the bits of three entries, but we know this will not
+// result in a collision. In the unlikely event that changes to
+// UnicodeData.txt introduce collisions, the compiler will catch it.
+// Note that the recomposition map for NFC and NFKC are identical.
+
+// combine returns the combined rune or 0 if it doesn't exist.
+func combine(a, b rune) rune {
+	key := uint32(uint16(a))<<16 + uint32(uint16(b))
+	return recompMap[key]
+}
+
+func lookupInfoNFC(b input, i int) Properties {
+	v, sz := b.charinfoNFC(i)
+	return compInfo(v, sz)
+}
+
+func lookupInfoNFKC(b input, i int) Properties {
+	v, sz := b.charinfoNFKC(i)
+	return compInfo(v, sz)
+}
+
+// Properties returns properties for the first rune in s.
+func (f Form) Properties(s []byte) Properties {
+	if f == NFC || f == NFD {
+		return compInfo(nfcTrie.lookup(s))
+	}
+	return compInfo(nfkcTrie.lookup(s))
+}
+
+// PropertiesString returns properties for the first rune in s.
+func (f Form) PropertiesString(s string) Properties {
+	if f == NFC || f == NFD {
+		return compInfo(nfcTrie.lookupString(s))
+	}
+	return compInfo(nfkcTrie.lookupString(s))
+}
+
+// compInfo converts the information contained in v and sz
+// to a Properties.  See the comment at the top of the file
+// for more information on the format.
+func compInfo(v uint16, sz int) Properties {
+	if v == 0 {
+		return Properties{size: uint8(sz)}
+	} else if v >= 0x8000 {
+		p := Properties{
+			size:  uint8(sz),
+			ccc:   uint8(v),
+			tccc:  uint8(v),
+			flags: qcInfo(v >> 8),
+		}
+		if p.ccc > 0 || p.combinesBackward() {
+			p.nLead = uint8(p.flags & 0x3)
+		}
+		return p
+	}
+	// has decomposition
+	h := decomps[v]
+	f := (qcInfo(h&headerFlagsMask) >> 2) | 0x4
+	p := Properties{size: uint8(sz), flags: f, index: v}
+	if v >= firstCCC {
+		v += uint16(h&headerLenMask) + 1
+		c := decomps[v]
+		p.tccc = c >> 2
+		p.flags |= qcInfo(c & 0x3)
+		if v >= firstLeadingCCC {
+			p.nLead = c & 0x3
+			if v >= firstStarterWithNLead {
+				// We were tricked. Remove the decomposition.
+				p.flags &= 0x03
+				p.index = 0
+				return p
+			}
+			p.ccc = decomps[v+1]
+		}
+	}
+	return p
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/forminfo_test.go

@@ -0,0 +1,54 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build test
+
+package norm
+
+import "testing"
+
+func TestProperties(t *testing.T) {
+	var d runeData
+	CK := [2]string{"C", "K"}
+	for k, r := 1, rune(0); r < 0x2ffff; r++ {
+		if k < len(testData) && r == testData[k].r {
+			d = testData[k]
+			k++
+		}
+		s := string(r)
+		for j, p := range []Properties{NFC.PropertiesString(s), NFKC.PropertiesString(s)} {
+			f := d.f[j]
+			if p.CCC() != d.ccc {
+				t.Errorf("%U: ccc(%s): was %d; want %d %X", r, CK[j], p.CCC(), d.ccc, p.index)
+			}
+			if p.isYesC() != (f.qc == Yes) {
+				t.Errorf("%U: YesC(%s): was %v; want %v", r, CK[j], p.isYesC(), f.qc == Yes)
+			}
+			if p.combinesBackward() != (f.qc == Maybe) {
+				t.Errorf("%U: combines backwards(%s): was %v; want %v", r, CK[j], p.combinesBackward(), f.qc == Maybe)
+			}
+			if p.nLeadingNonStarters() != d.nLead {
+				t.Errorf("%U: nLead(%s): was %d; want %d %#v %#v", r, CK[j], p.nLeadingNonStarters(), d.nLead, p, d)
+			}
+			if p.nTrailingNonStarters() != d.nTrail {
+				t.Errorf("%U: nTrail(%s): was %d; want %d %#v %#v", r, CK[j], p.nTrailingNonStarters(), d.nTrail, p, d)
+			}
+			if p.combinesForward() != f.combinesForward {
+				t.Errorf("%U: combines forward(%s): was %v; want %v %#v", r, CK[j], p.combinesForward(), f.combinesForward, p)
+			}
+			// Skip Hangul as it is algorithmically computed.
+			if r >= hangulBase && r < hangulEnd {
+				continue
+			}
+			if p.hasDecomposition() {
+				if has := f.decomposition != ""; !has {
+					t.Errorf("%U: hasDecomposition(%s): was %v; want %v", r, CK[j], p.hasDecomposition(), has)
+				}
+				if string(p.Decomposition()) != f.decomposition {
+					t.Errorf("%U: decomp(%s): was %+q; want %+q", r, CK[j], p.Decomposition(), f.decomposition)
+				}
+			}
+		}
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/input.go

@@ -0,0 +1,105 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "unicode/utf8"
+
+type input struct {
+	str   string
+	bytes []byte
+}
+
+func inputBytes(str []byte) input {
+	return input{bytes: str}
+}
+
+func inputString(str string) input {
+	return input{str: str}
+}
+
+func (in *input) setBytes(str []byte) {
+	in.str = ""
+	in.bytes = str
+}
+
+func (in *input) setString(str string) {
+	in.str = str
+	in.bytes = nil
+}
+
+func (in *input) _byte(p int) byte {
+	if in.bytes == nil {
+		return in.str[p]
+	}
+	return in.bytes[p]
+}
+
+func (in *input) skipASCII(p, max int) int {
+	if in.bytes == nil {
+		for ; p < max && in.str[p] < utf8.RuneSelf; p++ {
+		}
+	} else {
+		for ; p < max && in.bytes[p] < utf8.RuneSelf; p++ {
+		}
+	}
+	return p
+}
+
+func (in *input) skipContinuationBytes(p int) int {
+	if in.bytes == nil {
+		for ; p < len(in.str) && !utf8.RuneStart(in.str[p]); p++ {
+		}
+	} else {
+		for ; p < len(in.bytes) && !utf8.RuneStart(in.bytes[p]); p++ {
+		}
+	}
+	return p
+}
+
+func (in *input) appendSlice(buf []byte, b, e int) []byte {
+	if in.bytes != nil {
+		return append(buf, in.bytes[b:e]...)
+	}
+	for i := b; i < e; i++ {
+		buf = append(buf, in.str[i])
+	}
+	return buf
+}
+
+func (in *input) copySlice(buf []byte, b, e int) int {
+	if in.bytes == nil {
+		return copy(buf, in.str[b:e])
+	}
+	return copy(buf, in.bytes[b:e])
+}
+
+func (in *input) charinfoNFC(p int) (uint16, int) {
+	if in.bytes == nil {
+		return nfcTrie.lookupString(in.str[p:])
+	}
+	return nfcTrie.lookup(in.bytes[p:])
+}
+
+func (in *input) charinfoNFKC(p int) (uint16, int) {
+	if in.bytes == nil {
+		return nfkcTrie.lookupString(in.str[p:])
+	}
+	return nfkcTrie.lookup(in.bytes[p:])
+}
+
+func (in *input) hangul(p int) (r rune) {
+	if in.bytes == nil {
+		if !isHangulString(in.str[p:]) {
+			return 0
+		}
+		r, _ = utf8.DecodeRuneInString(in.str[p:])
+	} else {
+		if !isHangul(in.bytes[p:]) {
+			return 0
+		}
+		r, _ = utf8.DecodeRune(in.bytes[p:])
+	}
+	return r
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/iter.go

@@ -0,0 +1,448 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import (
+	"fmt"
+	"unicode/utf8"
+)
+
+const MaxSegmentSize = maxByteBufferSize
+
+// An Iter iterates over a string or byte slice, while normalizing it
+// to a given Form.
+type Iter struct {
+	rb     reorderBuffer
+	buf    [maxByteBufferSize]byte
+	info   Properties // first character saved from previous iteration
+	next   iterFunc   // implementation of next depends on form
+	asciiF iterFunc
+
+	p        int    // current position in input source
+	multiSeg []byte // remainder of multi-segment decomposition
+}
+
+type iterFunc func(*Iter) []byte
+
+// Init initializes i to iterate over src after normalizing it to Form f.
+func (i *Iter) Init(f Form, src []byte) {
+	i.p = 0
+	if len(src) == 0 {
+		i.setDone()
+		i.rb.nsrc = 0
+		return
+	}
+	i.multiSeg = nil
+	i.rb.init(f, src)
+	i.next = i.rb.f.nextMain
+	i.asciiF = nextASCIIBytes
+	i.info = i.rb.f.info(i.rb.src, i.p)
+}
+
+// InitString initializes i to iterate over src after normalizing it to Form f.
+func (i *Iter) InitString(f Form, src string) {
+	i.p = 0
+	if len(src) == 0 {
+		i.setDone()
+		i.rb.nsrc = 0
+		return
+	}
+	i.multiSeg = nil
+	i.rb.initString(f, src)
+	i.next = i.rb.f.nextMain
+	i.asciiF = nextASCIIString
+	i.info = i.rb.f.info(i.rb.src, i.p)
+}
+
+// Seek sets the segment to be returned by the next call to Next to start
+// at position p.  It is the responsibility of the caller to set p to the
+// start of a UTF8 rune.
+func (i *Iter) Seek(offset int64, whence int) (int64, error) {
+	var abs int64
+	switch whence {
+	case 0:
+		abs = offset
+	case 1:
+		abs = int64(i.p) + offset
+	case 2:
+		abs = int64(i.rb.nsrc) + offset
+	default:
+		return 0, fmt.Errorf("norm: invalid whence")
+	}
+	if abs < 0 {
+		return 0, fmt.Errorf("norm: negative position")
+	}
+	if int(abs) >= i.rb.nsrc {
+		i.setDone()
+		return int64(i.p), nil
+	}
+	i.p = int(abs)
+	i.multiSeg = nil
+	i.next = i.rb.f.nextMain
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	return abs, nil
+}
+
+// returnSlice returns a slice of the underlying input type as a byte slice.
+// If the underlying is of type []byte, it will simply return a slice.
+// If the underlying is of type string, it will copy the slice to the buffer
+// and return that.
+func (i *Iter) returnSlice(a, b int) []byte {
+	if i.rb.src.bytes == nil {
+		return i.buf[:copy(i.buf[:], i.rb.src.str[a:b])]
+	}
+	return i.rb.src.bytes[a:b]
+}
+
+// Pos returns the byte position at which the next call to Next will commence processing.
+func (i *Iter) Pos() int {
+	return i.p
+}
+
+func (i *Iter) setDone() {
+	i.next = nextDone
+	i.p = i.rb.nsrc
+}
+
+// Done returns true if there is no more input to process.
+func (i *Iter) Done() bool {
+	return i.p >= i.rb.nsrc
+}
+
+// Next returns f(i.input[i.Pos():n]), where n is a boundary of i.input.
+// For any input a and b for which f(a) == f(b), subsequent calls
+// to Next will return the same segments.
+// Modifying runes are grouped together with the preceding starter, if such a starter exists.
+// Although not guaranteed, n will typically be the smallest possible n.
+func (i *Iter) Next() []byte {
+	return i.next(i)
+}
+
+func nextASCIIBytes(i *Iter) []byte {
+	p := i.p + 1
+	if p >= i.rb.nsrc {
+		i.setDone()
+		return i.rb.src.bytes[i.p:p]
+	}
+	if i.rb.src.bytes[p] < utf8.RuneSelf {
+		p0 := i.p
+		i.p = p
+		return i.rb.src.bytes[p0:p]
+	}
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.next = i.rb.f.nextMain
+	return i.next(i)
+}
+
+func nextASCIIString(i *Iter) []byte {
+	p := i.p + 1
+	if p >= i.rb.nsrc {
+		i.buf[0] = i.rb.src.str[i.p]
+		i.setDone()
+		return i.buf[:1]
+	}
+	if i.rb.src.str[p] < utf8.RuneSelf {
+		i.buf[0] = i.rb.src.str[i.p]
+		i.p = p
+		return i.buf[:1]
+	}
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.next = i.rb.f.nextMain
+	return i.next(i)
+}
+
+func nextHangul(i *Iter) []byte {
+	p := i.p
+	next := p + hangulUTF8Size
+	if next >= i.rb.nsrc {
+		i.setDone()
+	} else if i.rb.src.hangul(next) == 0 {
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		i.next = i.rb.f.nextMain
+		return i.next(i)
+	}
+	i.p = next
+	return i.buf[:decomposeHangul(i.buf[:], i.rb.src.hangul(p))]
+}
+
+func nextDone(i *Iter) []byte {
+	return nil
+}
+
+// nextMulti is used for iterating over multi-segment decompositions
+// for decomposing normal forms.
+func nextMulti(i *Iter) []byte {
+	j := 0
+	d := i.multiSeg
+	// skip first rune
+	for j = 1; j < len(d) && !utf8.RuneStart(d[j]); j++ {
+	}
+	for j < len(d) {
+		info := i.rb.f.info(input{bytes: d}, j)
+		if info.BoundaryBefore() {
+			i.multiSeg = d[j:]
+			return d[:j]
+		}
+		j += int(info.size)
+	}
+	// treat last segment as normal decomposition
+	i.next = i.rb.f.nextMain
+	return i.next(i)
+}
+
+// nextMultiNorm is used for iterating over multi-segment decompositions
+// for composing normal forms.
+func nextMultiNorm(i *Iter) []byte {
+	j := 0
+	d := i.multiSeg
+	for j < len(d) {
+		info := i.rb.f.info(input{bytes: d}, j)
+		if info.BoundaryBefore() {
+			i.rb.compose()
+			seg := i.buf[:i.rb.flushCopy(i.buf[:])]
+			i.rb.ss.first(info)
+			i.rb.insertUnsafe(input{bytes: d}, j, info)
+			i.multiSeg = d[j+int(info.size):]
+			return seg
+		}
+		i.rb.ss.next(info)
+		i.rb.insertUnsafe(input{bytes: d}, j, info)
+		j += int(info.size)
+	}
+	i.multiSeg = nil
+	i.next = nextComposed
+	return doNormComposed(i)
+}
+
+// nextDecomposed is the implementation of Next for forms NFD and NFKD.
+func nextDecomposed(i *Iter) (next []byte) {
+	outp := 0
+	inCopyStart, outCopyStart := i.p, 0
+	ss := mkStreamSafe(i.info)
+	for {
+		if sz := int(i.info.size); sz <= 1 {
+			p := i.p
+			i.p++ // ASCII or illegal byte.  Either way, advance by 1.
+			if i.p >= i.rb.nsrc {
+				i.setDone()
+				return i.returnSlice(p, i.p)
+			} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
+				i.next = i.asciiF
+				return i.returnSlice(p, i.p)
+			}
+			outp++
+		} else if d := i.info.Decomposition(); d != nil {
+			// Note: If leading CCC != 0, then len(d) == 2 and last is also non-zero.
+			// Case 1: there is a leftover to copy.  In this case the decomposition
+			// must begin with a modifier and should always be appended.
+			// Case 2: no leftover. Simply return d if followed by a ccc == 0 value.
+			p := outp + len(d)
+			if outp > 0 {
+				i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
+				if p > len(i.buf) {
+					return i.buf[:outp]
+				}
+			} else if i.info.multiSegment() {
+				// outp must be 0 as multi-segment decompositions always
+				// start a new segment.
+				if i.multiSeg == nil {
+					i.multiSeg = d
+					i.next = nextMulti
+					return nextMulti(i)
+				}
+				// We are in the last segment.  Treat as normal decomposition.
+				d = i.multiSeg
+				i.multiSeg = nil
+				p = len(d)
+			}
+			prevCC := i.info.tccc
+			if i.p += sz; i.p >= i.rb.nsrc {
+				i.setDone()
+				i.info = Properties{} // Force BoundaryBefore to succeed.
+			} else {
+				i.info = i.rb.f.info(i.rb.src, i.p)
+			}
+			switch ss.next(i.info) {
+			case ssOverflow:
+				i.next = nextCGJDecompose
+				fallthrough
+			case ssStarter:
+				if outp > 0 {
+					copy(i.buf[outp:], d)
+					return i.buf[:p]
+				}
+				return d
+			}
+			copy(i.buf[outp:], d)
+			outp = p
+			inCopyStart, outCopyStart = i.p, outp
+			if i.info.ccc < prevCC {
+				goto doNorm
+			}
+			continue
+		} else if r := i.rb.src.hangul(i.p); r != 0 {
+			outp = decomposeHangul(i.buf[:], r)
+			i.p += hangulUTF8Size
+			inCopyStart, outCopyStart = i.p, outp
+			if i.p >= i.rb.nsrc {
+				i.setDone()
+				break
+			} else if i.rb.src.hangul(i.p) != 0 {
+				i.next = nextHangul
+				return i.buf[:outp]
+			}
+		} else {
+			p := outp + sz
+			if p > len(i.buf) {
+				break
+			}
+			outp = p
+			i.p += sz
+		}
+		if i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		}
+		prevCC := i.info.tccc
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if v := ss.next(i.info); v == ssStarter {
+			break
+		} else if v == ssOverflow {
+			i.next = nextCGJDecompose
+			break
+		}
+		if i.info.ccc < prevCC {
+			goto doNorm
+		}
+	}
+	if outCopyStart == 0 {
+		return i.returnSlice(inCopyStart, i.p)
+	} else if inCopyStart < i.p {
+		i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
+	}
+	return i.buf[:outp]
+doNorm:
+	// Insert what we have decomposed so far in the reorderBuffer.
+	// As we will only reorder, there will always be enough room.
+	i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
+	i.rb.insertDecomposed(i.buf[0:outp])
+	return doNormDecomposed(i)
+}
+
+func doNormDecomposed(i *Iter) []byte {
+	for {
+		if s := i.rb.ss.next(i.info); s == ssOverflow {
+			i.next = nextCGJDecompose
+			break
+		}
+		i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+		if i.p += int(i.info.size); i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		}
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if i.info.ccc == 0 {
+			break
+		}
+	}
+	// new segment or too many combining characters: exit normalization
+	return i.buf[:i.rb.flushCopy(i.buf[:])]
+}
+
+func nextCGJDecompose(i *Iter) []byte {
+	i.rb.ss = 0
+	i.rb.insertCGJ()
+	i.next = nextDecomposed
+	buf := doNormDecomposed(i)
+	return buf
+}
+
+// nextComposed is the implementation of Next for forms NFC and NFKC.
+func nextComposed(i *Iter) []byte {
+	outp, startp := 0, i.p
+	var prevCC uint8
+	ss := mkStreamSafe(i.info)
+	for {
+		if !i.info.isYesC() {
+			goto doNorm
+		}
+		prevCC = i.info.tccc
+		sz := int(i.info.size)
+		if sz == 0 {
+			sz = 1 // illegal rune: copy byte-by-byte
+		}
+		p := outp + sz
+		if p > len(i.buf) {
+			break
+		}
+		outp = p
+		i.p += sz
+		if i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
+			i.next = i.asciiF
+			break
+		}
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if v := ss.next(i.info); v == ssStarter {
+			break
+		} else if v == ssOverflow {
+			i.next = nextCGJCompose
+			break
+		}
+		if i.info.ccc < prevCC {
+			goto doNorm
+		}
+	}
+	return i.returnSlice(startp, i.p)
+doNorm:
+	i.p = startp
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	if i.info.multiSegment() {
+		d := i.info.Decomposition()
+		info := i.rb.f.info(input{bytes: d}, 0)
+		i.rb.insertUnsafe(input{bytes: d}, 0, info)
+		i.multiSeg = d[int(info.size):]
+		i.next = nextMultiNorm
+		return nextMultiNorm(i)
+	}
+	i.rb.ss.first(i.info)
+	i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+	return doNormComposed(i)
+}
+
+func doNormComposed(i *Iter) []byte {
+	// First rune should already be inserted.
+	for {
+		if i.p += int(i.info.size); i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		}
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if s := i.rb.ss.next(i.info); s == ssStarter {
+			break
+		} else if s == ssOverflow {
+			i.next = nextCGJCompose
+			break
+		}
+		i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+	}
+	i.rb.compose()
+	seg := i.buf[:i.rb.flushCopy(i.buf[:])]
+	return seg
+}
+
+func nextCGJCompose(i *Iter) []byte {
+	i.rb.ss = 0 // instead of first
+	i.rb.insertCGJ()
+	i.next = nextComposed
+	// Note that we treat any rune with nLeadingNonStarters > 0 as a non-starter,
+	// even if they are not. This is particularly dubious for U+FF9E and UFF9A.
+	// If we ever change that, insert a check here.
+	i.rb.ss.first(i.info)
+	i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+	return doNormComposed(i)
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/iter_test.go

@@ -0,0 +1,98 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import (
+	"strings"
+	"testing"
+)
+
+func doIterNorm(f Form, s string) []byte {
+	acc := []byte{}
+	i := Iter{}
+	i.InitString(f, s)
+	for !i.Done() {
+		acc = append(acc, i.Next()...)
+	}
+	return acc
+}
+
+func TestIterNext(t *testing.T) {
+	runNormTests(t, "IterNext", func(f Form, out []byte, s string) []byte {
+		return doIterNorm(f, string(append(out, s...)))
+	})
+}
+
+type SegmentTest struct {
+	in  string
+	out []string
+}
+
+var segmentTests = []SegmentTest{
+	{"\u1E0A\u0323a", []string{"\x44\u0323\u0307", "a", ""}},
+	{rep('a', segSize), append(strings.Split(rep('a', segSize), ""), "")},
+	{rep('a', segSize+2), append(strings.Split(rep('a', segSize+2), ""), "")},
+	{rep('a', segSize) + "\u0300aa",
+		append(strings.Split(rep('a', segSize-1), ""), "a\u0300", "a", "a", "")},
+
+	// U+0f73 is NOT treated as a starter as it is a modifier
+	{"a" + grave(29) + "\u0f73", []string{"a" + grave(29), cgj + "\u0f73"}},
+	{"a\u0f73", []string{"a\u0f73"}},
+
+	// U+ff9e is treated as a non-starter.
+	// TODO: should we? Note that this will only affect iteration, as whether
+	// or not we do so does not affect the normalization output and will either
+	// way result in consistent iteration output.
+	{"a" + grave(30) + "\uff9e", []string{"a" + grave(30), cgj + "\uff9e"}},
+	{"a\uff9e", []string{"a\uff9e"}},
+}
+
+var segmentTestsK = []SegmentTest{
+	{"\u3332", []string{"\u30D5", "\u30A1", "\u30E9", "\u30C3", "\u30C8\u3099", ""}},
+	// last segment of multi-segment decomposition needs normalization
+	{"\u3332\u093C", []string{"\u30D5", "\u30A1", "\u30E9", "\u30C3", "\u30C8\u093C\u3099", ""}},
+	{"\u320E", []string{"\x28", "\uAC00", "\x29"}},
+
+	// last segment should be copied to start of buffer.
+	{"\ufdfa", []string{"\u0635", "\u0644", "\u0649", " ", "\u0627", "\u0644", "\u0644", "\u0647", " ", "\u0639", "\u0644", "\u064a", "\u0647", " ", "\u0648", "\u0633", "\u0644", "\u0645", ""}},
+	{"\ufdfa" + grave(30), []string{"\u0635", "\u0644", "\u0649", " ", "\u0627", "\u0644", "\u0644", "\u0647", " ", "\u0639", "\u0644", "\u064a", "\u0647", " ", "\u0648", "\u0633", "\u0644", "\u0645" + grave(30), ""}},
+	{"\uFDFA" + grave(64), []string{"\u0635", "\u0644", "\u0649", " ", "\u0627", "\u0644", "\u0644", "\u0647", " ", "\u0639", "\u0644", "\u064a", "\u0647", " ", "\u0648", "\u0633", "\u0644", "\u0645" + grave(30), cgj + grave(30), cgj + grave(4), ""}},
+
+	// Hangul and Jamo are grouped togeter.
+	{"\uAC00", []string{"\u1100\u1161", ""}},
+	{"\uAC01", []string{"\u1100\u1161\u11A8", ""}},
+	{"\u1100\u1161", []string{"\u1100\u1161", ""}},
+}
+
+// Note that, by design, segmentation is equal for composing and decomposing forms.
+func TestIterSegmentation(t *testing.T) {
+	segmentTest(t, "SegmentTestD", NFD, segmentTests)
+	segmentTest(t, "SegmentTestC", NFC, segmentTests)
+	segmentTest(t, "SegmentTestKD", NFKD, segmentTestsK)
+	segmentTest(t, "SegmentTestKC", NFKC, segmentTestsK)
+}
+
+func segmentTest(t *testing.T, name string, f Form, tests []SegmentTest) {
+	iter := Iter{}
+	for i, tt := range tests {
+		iter.InitString(f, tt.in)
+		for j, seg := range tt.out {
+			if seg == "" {
+				if !iter.Done() {
+					res := string(iter.Next())
+					t.Errorf(`%s:%d:%d: expected Done()==true, found segment %+q`, name, i, j, res)
+				}
+				continue
+			}
+			if iter.Done() {
+				t.Errorf("%s:%d:%d: Done()==true, want false", name, i, j)
+			}
+			seg = f.String(seg)
+			if res := string(iter.Next()); res != seg {
+				t.Errorf(`%s:%d:%d" segment was %+q (%d); want %+q (%d)`, name, i, j, pc(res), len(res), pc(seg), len(seg))
+			}
+		}
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/maketesttables.go

@@ -0,0 +1,45 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Generate test data for trie code.
+
+package main
+
+import (
+	"fmt"
+)
+
+func main() {
+	printTestTables()
+}
+
+// We take the smallest, largest and an arbitrary value for each
+// of the UTF-8 sequence lengths.
+var testRunes = []rune{
+	0x01, 0x0C, 0x7F, // 1-byte sequences
+	0x80, 0x100, 0x7FF, // 2-byte sequences
+	0x800, 0x999, 0xFFFF, // 3-byte sequences
+	0x10000, 0x10101, 0x10FFFF, // 4-byte sequences
+	0x200, 0x201, 0x202, 0x210, 0x215, // five entries in one sparse block
+}
+
+const fileHeader = `// Generated by running
+//	maketesttables
+// DO NOT EDIT
+
+package norm
+
+`
+
+func printTestTables() {
+	fmt.Print(fileHeader)
+	fmt.Printf("var testRunes = %#v\n\n", testRunes)
+	t := newNode()
+	for i, r := range testRunes {
+		t.insert(r, uint16(i))
+	}
+	t.printTables("testdata")
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/norm_test.go

@@ -0,0 +1,14 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm_test
+
+import (
+	"testing"
+)
+
+func TestPlaceHolder(t *testing.T) {
+	// Does nothing, just allows the Makefile to be canonical
+	// while waiting for the package itself to be written.
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/normalize.go

@@ -0,0 +1,524 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package norm contains types and functions for normalizing Unicode strings.
+package norm
+
+import "unicode/utf8"
+
+// A Form denotes a canonical representation of Unicode code points.
+// The Unicode-defined normalization and equivalence forms are:
+//
+//   NFC   Unicode Normalization Form C
+//   NFD   Unicode Normalization Form D
+//   NFKC  Unicode Normalization Form KC
+//   NFKD  Unicode Normalization Form KD
+//
+// For a Form f, this documentation uses the notation f(x) to mean
+// the bytes or string x converted to the given form.
+// A position n in x is called a boundary if conversion to the form can
+// proceed independently on both sides:
+//   f(x) == append(f(x[0:n]), f(x[n:])...)
+//
+// References: http://unicode.org/reports/tr15/ and
+// http://unicode.org/notes/tn5/.
+type Form int
+
+const (
+	NFC Form = iota
+	NFD
+	NFKC
+	NFKD
+)
+
+// Bytes returns f(b). May return b if f(b) = b.
+func (f Form) Bytes(b []byte) []byte {
+	src := inputBytes(b)
+	ft := formTable[f]
+	n, ok := ft.quickSpan(src, 0, len(b), true)
+	if ok {
+		return b
+	}
+	out := make([]byte, n, len(b))
+	copy(out, b[0:n])
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(b), out: out, flushF: appendFlush}
+	return doAppendInner(&rb, n)
+}
+
+// String returns f(s).
+func (f Form) String(s string) string {
+	src := inputString(s)
+	ft := formTable[f]
+	n, ok := ft.quickSpan(src, 0, len(s), true)
+	if ok {
+		return s
+	}
+	out := make([]byte, n, len(s))
+	copy(out, s[0:n])
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(s), out: out, flushF: appendFlush}
+	return string(doAppendInner(&rb, n))
+}
+
+// IsNormal returns true if b == f(b).
+func (f Form) IsNormal(b []byte) bool {
+	src := inputBytes(b)
+	ft := formTable[f]
+	bp, ok := ft.quickSpan(src, 0, len(b), true)
+	if ok {
+		return true
+	}
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(b)}
+	rb.setFlusher(nil, cmpNormalBytes)
+	for bp < len(b) {
+		rb.out = b[bp:]
+		if bp = decomposeSegment(&rb, bp, true); bp < 0 {
+			return false
+		}
+		bp, _ = rb.f.quickSpan(rb.src, bp, len(b), true)
+	}
+	return true
+}
+
+func cmpNormalBytes(rb *reorderBuffer) bool {
+	b := rb.out
+	for i := 0; i < rb.nrune; i++ {
+		info := rb.rune[i]
+		if int(info.size) > len(b) {
+			return false
+		}
+		p := info.pos
+		pe := p + info.size
+		for ; p < pe; p++ {
+			if b[0] != rb.byte[p] {
+				return false
+			}
+			b = b[1:]
+		}
+	}
+	return true
+}
+
+// IsNormalString returns true if s == f(s).
+func (f Form) IsNormalString(s string) bool {
+	src := inputString(s)
+	ft := formTable[f]
+	bp, ok := ft.quickSpan(src, 0, len(s), true)
+	if ok {
+		return true
+	}
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(s)}
+	rb.setFlusher(nil, func(rb *reorderBuffer) bool {
+		for i := 0; i < rb.nrune; i++ {
+			info := rb.rune[i]
+			if bp+int(info.size) > len(s) {
+				return false
+			}
+			p := info.pos
+			pe := p + info.size
+			for ; p < pe; p++ {
+				if s[bp] != rb.byte[p] {
+					return false
+				}
+				bp++
+			}
+		}
+		return true
+	})
+	for bp < len(s) {
+		if bp = decomposeSegment(&rb, bp, true); bp < 0 {
+			return false
+		}
+		bp, _ = rb.f.quickSpan(rb.src, bp, len(s), true)
+	}
+	return true
+}
+
+// patchTail fixes a case where a rune may be incorrectly normalized
+// if it is followed by illegal continuation bytes. It returns the
+// patched buffer and whether the decomposition is still in progress.
+func patchTail(rb *reorderBuffer) bool {
+	info, p := lastRuneStart(&rb.f, rb.out)
+	if p == -1 || info.size == 0 {
+		return true
+	}
+	end := p + int(info.size)
+	extra := len(rb.out) - end
+	if extra > 0 {
+		// Potentially allocating memory. However, this only
+		// happens with ill-formed UTF-8.
+		x := make([]byte, 0)
+		x = append(x, rb.out[len(rb.out)-extra:]...)
+		rb.out = rb.out[:end]
+		decomposeToLastBoundary(rb)
+		rb.doFlush()
+		rb.out = append(rb.out, x...)
+		return false
+	}
+	buf := rb.out[p:]
+	rb.out = rb.out[:p]
+	decomposeToLastBoundary(rb)
+	if s := rb.ss.next(info); s == ssStarter {
+		rb.doFlush()
+		rb.ss.first(info)
+	} else if s == ssOverflow {
+		rb.doFlush()
+		rb.insertCGJ()
+		rb.ss = 0
+	}
+	rb.insertUnsafe(inputBytes(buf), 0, info)
+	return true
+}
+
+func appendQuick(rb *reorderBuffer, i int) int {
+	if rb.nsrc == i {
+		return i
+	}
+	end, _ := rb.f.quickSpan(rb.src, i, rb.nsrc, true)
+	rb.out = rb.src.appendSlice(rb.out, i, end)
+	return end
+}
+
+// Append returns f(append(out, b...)).
+// The buffer out must be nil, empty, or equal to f(out).
+func (f Form) Append(out []byte, src ...byte) []byte {
+	return f.doAppend(out, inputBytes(src), len(src))
+}
+
+func (f Form) doAppend(out []byte, src input, n int) []byte {
+	if n == 0 {
+		return out
+	}
+	ft := formTable[f]
+	// Attempt to do a quickSpan first so we can avoid initializing the reorderBuffer.
+	if len(out) == 0 {
+		p, _ := ft.quickSpan(src, 0, n, true)
+		out = src.appendSlice(out, 0, p)
+		if p == n {
+			return out
+		}
+		rb := reorderBuffer{f: *ft, src: src, nsrc: n, out: out, flushF: appendFlush}
+		return doAppendInner(&rb, p)
+	}
+	rb := reorderBuffer{f: *ft, src: src, nsrc: n}
+	return doAppend(&rb, out, 0)
+}
+
+func doAppend(rb *reorderBuffer, out []byte, p int) []byte {
+	rb.setFlusher(out, appendFlush)
+	src, n := rb.src, rb.nsrc
+	doMerge := len(out) > 0
+	if q := src.skipContinuationBytes(p); q > p {
+		// Move leading non-starters to destination.
+		rb.out = src.appendSlice(rb.out, p, q)
+		p = q
+		doMerge = patchTail(rb)
+	}
+	fd := &rb.f
+	if doMerge {
+		var info Properties
+		if p < n {
+			info = fd.info(src, p)
+			if !info.BoundaryBefore() || info.nLeadingNonStarters() > 0 {
+				if p == 0 {
+					decomposeToLastBoundary(rb)
+				}
+				p = decomposeSegment(rb, p, true)
+			}
+		}
+		if info.size == 0 {
+			rb.doFlush()
+			// Append incomplete UTF-8 encoding.
+			return src.appendSlice(rb.out, p, n)
+		}
+		if rb.nrune > 0 {
+			return doAppendInner(rb, p)
+		}
+	}
+	p = appendQuick(rb, p)
+	return doAppendInner(rb, p)
+}
+
+func doAppendInner(rb *reorderBuffer, p int) []byte {
+	for n := rb.nsrc; p < n; {
+		p = decomposeSegment(rb, p, true)
+		p = appendQuick(rb, p)
+	}
+	return rb.out
+}
+
+// AppendString returns f(append(out, []byte(s))).
+// The buffer out must be nil, empty, or equal to f(out).
+func (f Form) AppendString(out []byte, src string) []byte {
+	return f.doAppend(out, inputString(src), len(src))
+}
+
+// QuickSpan returns a boundary n such that b[0:n] == f(b[0:n]).
+// It is not guaranteed to return the largest such n.
+func (f Form) QuickSpan(b []byte) int {
+	n, _ := formTable[f].quickSpan(inputBytes(b), 0, len(b), true)
+	return n
+}
+
+// quickSpan returns a boundary n such that src[0:n] == f(src[0:n]) and
+// whether any non-normalized parts were found. If atEOF is false, n will
+// not point past the last segment if this segment might be become
+// non-normalized by appending other runes.
+func (f *formInfo) quickSpan(src input, i, end int, atEOF bool) (n int, ok bool) {
+	var lastCC uint8
+	ss := streamSafe(0)
+	lastSegStart := i
+	for n = end; i < n; {
+		if j := src.skipASCII(i, n); i != j {
+			i = j
+			lastSegStart = i - 1
+			lastCC = 0
+			ss = 0
+			continue
+		}
+		info := f.info(src, i)
+		if info.size == 0 {
+			if atEOF {
+				// include incomplete runes
+				return n, true
+			}
+			return lastSegStart, true
+		}
+		// This block needs to be before the next, because it is possible to
+		// have an overflow for runes that are starters (e.g. with U+FF9E).
+		switch ss.next(info) {
+		case ssStarter:
+			ss.first(info)
+			lastSegStart = i
+		case ssOverflow:
+			return lastSegStart, false
+		case ssSuccess:
+			if lastCC > info.ccc {
+				return lastSegStart, false
+			}
+		}
+		if f.composing {
+			if !info.isYesC() {
+				break
+			}
+		} else {
+			if !info.isYesD() {
+				break
+			}
+		}
+		lastCC = info.ccc
+		i += int(info.size)
+	}
+	if i == n {
+		if !atEOF {
+			n = lastSegStart
+		}
+		return n, true
+	}
+	return lastSegStart, false
+}
+
+// QuickSpanString returns a boundary n such that b[0:n] == f(s[0:n]).
+// It is not guaranteed to return the largest such n.
+func (f Form) QuickSpanString(s string) int {
+	n, _ := formTable[f].quickSpan(inputString(s), 0, len(s), true)
+	return n
+}
+
+// FirstBoundary returns the position i of the first boundary in b
+// or -1 if b contains no boundary.
+func (f Form) FirstBoundary(b []byte) int {
+	return f.firstBoundary(inputBytes(b), len(b))
+}
+
+func (f Form) firstBoundary(src input, nsrc int) int {
+	i := src.skipContinuationBytes(0)
+	if i >= nsrc {
+		return -1
+	}
+	fd := formTable[f]
+	ss := streamSafe(0)
+	// We should call ss.first here, but we can't as the first rune is
+	// skipped already. This means FirstBoundary can't really determine
+	// CGJ insertion points correctly. Luckily it doesn't have to.
+	// TODO: consider adding NextBoundary
+	for {
+		info := fd.info(src, i)
+		if info.size == 0 {
+			return -1
+		}
+		if s := ss.next(info); s != ssSuccess {
+			return i
+		}
+		i += int(info.size)
+		if i >= nsrc {
+			if !info.BoundaryAfter() && !ss.isMax() {
+				return -1
+			}
+			return nsrc
+		}
+	}
+}
+
+// FirstBoundaryInString returns the position i of the first boundary in s
+// or -1 if s contains no boundary.
+func (f Form) FirstBoundaryInString(s string) int {
+	return f.firstBoundary(inputString(s), len(s))
+}
+
+// LastBoundary returns the position i of the last boundary in b
+// or -1 if b contains no boundary.
+func (f Form) LastBoundary(b []byte) int {
+	return lastBoundary(formTable[f], b)
+}
+
+func lastBoundary(fd *formInfo, b []byte) int {
+	i := len(b)
+	info, p := lastRuneStart(fd, b)
+	if p == -1 {
+		return -1
+	}
+	if info.size == 0 { // ends with incomplete rune
+		if p == 0 { // starts with incomplete rune
+			return -1
+		}
+		i = p
+		info, p = lastRuneStart(fd, b[:i])
+		if p == -1 { // incomplete UTF-8 encoding or non-starter bytes without a starter
+			return i
+		}
+	}
+	if p+int(info.size) != i { // trailing non-starter bytes: illegal UTF-8
+		return i
+	}
+	if info.BoundaryAfter() {
+		return i
+	}
+	ss := streamSafe(0)
+	v := ss.backwards(info)
+	for i = p; i >= 0 && v != ssStarter; i = p {
+		info, p = lastRuneStart(fd, b[:i])
+		if v = ss.backwards(info); v == ssOverflow {
+			break
+		}
+		if p+int(info.size) != i {
+			if p == -1 { // no boundary found
+				return -1
+			}
+			return i // boundary after an illegal UTF-8 encoding
+		}
+	}
+	return i
+}
+
+// decomposeSegment scans the first segment in src into rb. It inserts 0x034f
+// (Grapheme Joiner) when it encounters a sequence of more than 30 non-starters
+// and returns the number of bytes consumed from src or iShortDst or iShortSrc.
+func decomposeSegment(rb *reorderBuffer, sp int, atEOF bool) int {
+	// Force one character to be consumed.
+	info := rb.f.info(rb.src, sp)
+	if info.size == 0 {
+		return 0
+	}
+	if rb.nrune > 0 {
+		if s := rb.ss.next(info); s == ssStarter {
+			goto end
+		} else if s == ssOverflow {
+			rb.insertCGJ()
+			goto end
+		}
+	} else {
+		rb.ss.first(info)
+	}
+	if err := rb.insertFlush(rb.src, sp, info); err != iSuccess {
+		return int(err)
+	}
+	for {
+		sp += int(info.size)
+		if sp >= rb.nsrc {
+			if !atEOF && !info.BoundaryAfter() {
+				return int(iShortSrc)
+			}
+			break
+		}
+		info = rb.f.info(rb.src, sp)
+		if info.size == 0 {
+			if !atEOF {
+				return int(iShortSrc)
+			}
+			break
+		}
+		if s := rb.ss.next(info); s == ssStarter {
+			break
+		} else if s == ssOverflow {
+			rb.insertCGJ()
+			break
+		}
+		if err := rb.insertFlush(rb.src, sp, info); err != iSuccess {
+			return int(err)
+		}
+	}
+end:
+	if !rb.doFlush() {
+		return int(iShortDst)
+	}
+	return sp
+}
+
+// lastRuneStart returns the runeInfo and position of the last
+// rune in buf or the zero runeInfo and -1 if no rune was found.
+func lastRuneStart(fd *formInfo, buf []byte) (Properties, int) {
+	p := len(buf) - 1
+	for ; p >= 0 && !utf8.RuneStart(buf[p]); p-- {
+	}
+	if p < 0 {
+		return Properties{}, -1
+	}
+	return fd.info(inputBytes(buf), p), p
+}
+
+// decomposeToLastBoundary finds an open segment at the end of the buffer
+// and scans it into rb. Returns the buffer minus the last segment.
+func decomposeToLastBoundary(rb *reorderBuffer) {
+	fd := &rb.f
+	info, i := lastRuneStart(fd, rb.out)
+	if int(info.size) != len(rb.out)-i {
+		// illegal trailing continuation bytes
+		return
+	}
+	if info.BoundaryAfter() {
+		return
+	}
+	var add [maxNonStarters + 1]Properties // stores runeInfo in reverse order
+	padd := 0
+	ss := streamSafe(0)
+	p := len(rb.out)
+	for {
+		add[padd] = info
+		v := ss.backwards(info)
+		if v == ssOverflow {
+			// Note that if we have an overflow, it the string we are appending to
+			// is not correctly normalized. In this case the behavior is undefined.
+			break
+		}
+		padd++
+		p -= int(info.size)
+		if v == ssStarter || p < 0 {
+			break
+		}
+		info, i = lastRuneStart(fd, rb.out[:p])
+		if int(info.size) != p-i {
+			break
+		}
+	}
+	rb.ss = ss
+	// Copy bytes for insertion as we may need to overwrite rb.out.
+	var buf [maxBufferSize * utf8.UTFMax]byte
+	cp := buf[:copy(buf[:], rb.out[p:])]
+	rb.out = rb.out[:p]
+	for padd--; padd >= 0; padd-- {
+		info = add[padd]
+		rb.insertUnsafe(inputBytes(cp), 0, info)
+		cp = cp[info.size:]
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/normregtest.go

@@ -0,0 +1,318 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"regexp"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+	"unicode"
+	"unicode/utf8"
+
+	"code.google.com/p/go.text/unicode/norm"
+)
+
+func main() {
+	flag.Parse()
+	loadTestData()
+	CharacterByCharacterTests()
+	StandardTests()
+	PerformanceTest()
+	if errorCount == 0 {
+		fmt.Println("PASS")
+	}
+}
+
+const file = "NormalizationTest.txt"
+
+var url = flag.String("url",
+	"http://www.unicode.org/Public/"+unicode.Version+"/ucd/"+file,
+	"URL of Unicode database directory")
+var localFiles = flag.Bool("local",
+	false,
+	"data files have been copied to the current directory; for debugging only")
+
+var logger = log.New(os.Stderr, "", log.Lshortfile)
+
+// This regression test runs the test set in NormalizationTest.txt
+// (taken from http://www.unicode.org/Public/<unicode.Version>/ucd/).
+//
+// NormalizationTest.txt has form:
+// @Part0 # Specific cases
+// #
+// 1E0A;1E0A;0044 0307;1E0A;0044 0307; # (Ḋ; Ḋ; D◌̇; Ḋ; D◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE
+// 1E0C;1E0C;0044 0323;1E0C;0044 0323; # (Ḍ; Ḍ; D◌̣; Ḍ; D◌̣; ) LATIN CAPITAL LETTER D WITH DOT BELOW
+//
+// Each test has 5 columns (c1, c2, c3, c4, c5), where
+// (c1, c2, c3, c4, c5) == (c1, NFC(c1), NFD(c1), NFKC(c1), NFKD(c1))
+//
+// CONFORMANCE:
+// 1. The following invariants must be true for all conformant implementations
+//
+//    NFC
+//      c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3)
+//      c4 ==  NFC(c4) ==  NFC(c5)
+//
+//    NFD
+//      c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3)
+//      c5 ==  NFD(c4) ==  NFD(c5)
+//
+//    NFKC
+//      c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
+//
+//    NFKD
+//      c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
+//
+// 2. For every code point X assigned in this version of Unicode that is not
+//    specifically listed in Part 1, the following invariants must be true
+//    for all conformant implementations:
+//
+//      X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X)
+//
+
+// Column types.
+const (
+	cRaw = iota
+	cNFC
+	cNFD
+	cNFKC
+	cNFKD
+	cMaxColumns
+)
+
+// Holds data from NormalizationTest.txt
+var part []Part
+
+type Part struct {
+	name   string
+	number int
+	tests  []Test
+}
+
+type Test struct {
+	name   string
+	partnr int
+	number int
+	r      rune                // used for character by character test
+	cols   [cMaxColumns]string // Each has 5 entries, see below.
+}
+
+func (t Test) Name() string {
+	if t.number < 0 {
+		return part[t.partnr].name
+	}
+	return fmt.Sprintf("%s:%d", part[t.partnr].name, t.number)
+}
+
+var partRe = regexp.MustCompile(`@Part(\d) # (.*)$`)
+var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.*)$`)
+
+var counter int
+
+// Load the data form NormalizationTest.txt
+func loadTestData() {
+	if *localFiles {
+		pwd, _ := os.Getwd()
+		*url = "file://" + path.Join(pwd, file)
+	}
+	t := &http.Transport{}
+	t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
+	c := &http.Client{Transport: t}
+	resp, err := c.Get(*url)
+	if err != nil {
+		logger.Fatal(err)
+	}
+	if resp.StatusCode != 200 {
+		logger.Fatal("bad GET status for "+file, resp.Status)
+	}
+	f := resp.Body
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) == 0 || line[0] == '#' {
+			continue
+		}
+		m := partRe.FindStringSubmatch(line)
+		if m != nil {
+			if len(m) < 3 {
+				logger.Fatal("Failed to parse Part: ", line)
+			}
+			i, err := strconv.Atoi(m[1])
+			if err != nil {
+				logger.Fatal(err)
+			}
+			name := m[2]
+			part = append(part, Part{name: name[:len(name)-1], number: i})
+			continue
+		}
+		m = testRe.FindStringSubmatch(line)
+		if m == nil || len(m) < 7 {
+			logger.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
+		}
+		test := Test{name: m[6], partnr: len(part) - 1, number: counter}
+		counter++
+		for j := 1; j < len(m)-1; j++ {
+			for _, split := range strings.Split(m[j], " ") {
+				r, err := strconv.ParseUint(split, 16, 64)
+				if err != nil {
+					logger.Fatal(err)
+				}
+				if test.r == 0 {
+					// save for CharacterByCharacterTests
+					test.r = rune(r)
+				}
+				var buf [utf8.UTFMax]byte
+				sz := utf8.EncodeRune(buf[:], rune(r))
+				test.cols[j-1] += string(buf[:sz])
+			}
+		}
+		part := &part[len(part)-1]
+		part.tests = append(part.tests, test)
+	}
+	if scanner.Err() != nil {
+		logger.Fatal(scanner.Err())
+	}
+}
+
+var fstr = []string{"NFC", "NFD", "NFKC", "NFKD"}
+
+var errorCount int
+
+func cmpResult(t *Test, name string, f norm.Form, gold, test, result string) {
+	if gold != result {
+		errorCount++
+		if errorCount > 20 {
+			return
+		}
+		logger.Printf("%s:%s: %s(%+q)=%+q; want %+q: %s",
+			t.Name(), name, fstr[f], test, result, gold, t.name)
+	}
+}
+
+func cmpIsNormal(t *Test, name string, f norm.Form, test string, result, want bool) {
+	if result != want {
+		errorCount++
+		if errorCount > 20 {
+			return
+		}
+		logger.Printf("%s:%s: %s(%+q)=%v; want %v", t.Name(), name, fstr[f], test, result, want)
+	}
+}
+
+func doTest(t *Test, f norm.Form, gold, test string) {
+	testb := []byte(test)
+	result := f.Bytes(testb)
+	cmpResult(t, "Bytes", f, gold, test, string(result))
+
+	sresult := f.String(test)
+	cmpResult(t, "String", f, gold, test, sresult)
+
+	acc := []byte{}
+	i := norm.Iter{}
+	i.InitString(f, test)
+	for !i.Done() {
+		acc = append(acc, i.Next()...)
+	}
+	cmpResult(t, "Iter.Next", f, gold, test, string(acc))
+
+	buf := make([]byte, 128)
+	acc = nil
+	for p := 0; p < len(testb); {
+		nDst, nSrc, _ := f.Transform(buf, testb[p:], true)
+		acc = append(acc, buf[:nDst]...)
+		p += nSrc
+	}
+	cmpResult(t, "Transform", f, gold, test, string(acc))
+
+	for i := range test {
+		out := f.Append(f.Bytes([]byte(test[:i])), []byte(test[i:])...)
+		cmpResult(t, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out))
+	}
+	cmpIsNormal(t, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold)
+	cmpIsNormal(t, "IsNormalString", f, test, f.IsNormalString(test), test == gold)
+}
+
+func doConformanceTests(t *Test, partn int) {
+	for i := 0; i <= 2; i++ {
+		doTest(t, norm.NFC, t.cols[1], t.cols[i])
+		doTest(t, norm.NFD, t.cols[2], t.cols[i])
+		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
+		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
+	}
+	for i := 3; i <= 4; i++ {
+		doTest(t, norm.NFC, t.cols[3], t.cols[i])
+		doTest(t, norm.NFD, t.cols[4], t.cols[i])
+		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
+		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
+	}
+}
+
+func CharacterByCharacterTests() {
+	tests := part[1].tests
+	var last rune = 0
+	for i := 0; i <= len(tests); i++ { // last one is special case
+		var r rune
+		if i == len(tests) {
+			r = 0x2FA1E // Don't have to go to 0x10FFFF
+		} else {
+			r = tests[i].r
+		}
+		for last++; last < r; last++ {
+			// Check all characters that were not explicitly listed in the test.
+			t := &Test{partnr: 1, number: -1}
+			char := string(last)
+			doTest(t, norm.NFC, char, char)
+			doTest(t, norm.NFD, char, char)
+			doTest(t, norm.NFKC, char, char)
+			doTest(t, norm.NFKD, char, char)
+		}
+		if i < len(tests) {
+			doConformanceTests(&tests[i], 1)
+		}
+	}
+}
+
+func StandardTests() {
+	for _, j := range []int{0, 2, 3} {
+		for _, test := range part[j].tests {
+			doConformanceTests(&test, j)
+		}
+	}
+}
+
+// PerformanceTest verifies that normalization is O(n). If any of the
+// code does not properly check for maxCombiningChars, normalization
+// may exhibit O(n**2) behavior.
+func PerformanceTest() {
+	runtime.GOMAXPROCS(2)
+	success := make(chan bool, 1)
+	go func() {
+		buf := bytes.Repeat([]byte("\u035D"), 1024*1024)
+		buf = append(buf, "\u035B"...)
+		norm.NFC.Append(nil, buf...)
+		success <- true
+	}()
+	timeout := time.After(1 * time.Second)
+	select {
+	case <-success:
+		// test completed before the timeout
+	case <-timeout:
+		errorCount++
+		logger.Printf(`unexpectedly long time to complete PerformanceTest`)
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/readwriter.go

@@ -0,0 +1,126 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "io"
+
+type normWriter struct {
+	rb  reorderBuffer
+	w   io.Writer
+	buf []byte
+}
+
+// Write implements the standard write interface.  If the last characters are
+// not at a normalization boundary, the bytes will be buffered for the next
+// write. The remaining bytes will be written on close.
+func (w *normWriter) Write(data []byte) (n int, err error) {
+	// Process data in pieces to keep w.buf size bounded.
+	const chunk = 4000
+
+	for len(data) > 0 {
+		// Normalize into w.buf.
+		m := len(data)
+		if m > chunk {
+			m = chunk
+		}
+		w.rb.src = inputBytes(data[:m])
+		w.rb.nsrc = m
+		w.buf = doAppend(&w.rb, w.buf, 0)
+		data = data[m:]
+		n += m
+
+		// Write out complete prefix, save remainder.
+		// Note that lastBoundary looks back at most 31 runes.
+		i := lastBoundary(&w.rb.f, w.buf)
+		if i == -1 {
+			i = 0
+		}
+		if i > 0 {
+			if _, err = w.w.Write(w.buf[:i]); err != nil {
+				break
+			}
+			bn := copy(w.buf, w.buf[i:])
+			w.buf = w.buf[:bn]
+		}
+	}
+	return n, err
+}
+
+// Close forces data that remains in the buffer to be written.
+func (w *normWriter) Close() error {
+	if len(w.buf) > 0 {
+		_, err := w.w.Write(w.buf)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Writer returns a new writer that implements Write(b)
+// by writing f(b) to w.  The returned writer may use an
+// an internal buffer to maintain state across Write calls.
+// Calling its Close method writes any buffered data to w.
+func (f Form) Writer(w io.Writer) io.WriteCloser {
+	wr := &normWriter{rb: reorderBuffer{}, w: w}
+	wr.rb.init(f, nil)
+	return wr
+}
+
+type normReader struct {
+	rb           reorderBuffer
+	r            io.Reader
+	inbuf        []byte
+	outbuf       []byte
+	bufStart     int
+	lastBoundary int
+	err          error
+}
+
+// Read implements the standard read interface.
+func (r *normReader) Read(p []byte) (int, error) {
+	for {
+		if r.lastBoundary-r.bufStart > 0 {
+			n := copy(p, r.outbuf[r.bufStart:r.lastBoundary])
+			r.bufStart += n
+			if r.lastBoundary-r.bufStart > 0 {
+				return n, nil
+			}
+			return n, r.err
+		}
+		if r.err != nil {
+			return 0, r.err
+		}
+		outn := copy(r.outbuf, r.outbuf[r.lastBoundary:])
+		r.outbuf = r.outbuf[0:outn]
+		r.bufStart = 0
+
+		n, err := r.r.Read(r.inbuf)
+		r.rb.src = inputBytes(r.inbuf[0:n])
+		r.rb.nsrc, r.err = n, err
+		if n > 0 {
+			r.outbuf = doAppend(&r.rb, r.outbuf, 0)
+		}
+		if err == io.EOF {
+			r.lastBoundary = len(r.outbuf)
+		} else {
+			r.lastBoundary = lastBoundary(&r.rb.f, r.outbuf)
+			if r.lastBoundary == -1 {
+				r.lastBoundary = 0
+			}
+		}
+	}
+	panic("should not reach here")
+}
+
+// Reader returns a new reader that implements Read
+// by reading data from r and returning f(data).
+func (f Form) Reader(r io.Reader) io.Reader {
+	const chunk = 4000
+	buf := make([]byte, chunk)
+	rr := &normReader{rb: reorderBuffer{}, r: r, inbuf: buf}
+	rr.rb.init(f, buf)
+	return rr
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/readwriter_test.go

@@ -0,0 +1,56 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+)
+
+var bufSizes = []int{1, 2, 3, 4, 5, 6, 7, 8, 100, 101, 102, 103, 4000, 4001, 4002, 4003}
+
+func readFunc(size int) appendFunc {
+	return func(f Form, out []byte, s string) []byte {
+		out = append(out, s...)
+		r := f.Reader(bytes.NewBuffer(out))
+		buf := make([]byte, size)
+		result := []byte{}
+		for n, err := 0, error(nil); err == nil; {
+			n, err = r.Read(buf)
+			result = append(result, buf[:n]...)
+		}
+		return result
+	}
+}
+
+func TestReader(t *testing.T) {
+	for _, s := range bufSizes {
+		name := fmt.Sprintf("TestReader%d", s)
+		runNormTests(t, name, readFunc(s))
+	}
+}
+
+func writeFunc(size int) appendFunc {
+	return func(f Form, out []byte, s string) []byte {
+		in := append(out, s...)
+		result := new(bytes.Buffer)
+		w := f.Writer(result)
+		buf := make([]byte, size)
+		for n := 0; len(in) > 0; in = in[n:] {
+			n = copy(buf, in)
+			_, _ = w.Write(buf[:n])
+		}
+		w.Close()
+		return result.Bytes()
+	}
+}
+
+func TestWriter(t *testing.T) {
+	for _, s := range bufSizes {
+		name := fmt.Sprintf("TestWriter%d", s)
+		runNormTests(t, name, writeFunc(s))
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/transform.go

@@ -0,0 +1,85 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import (
+	"unicode/utf8"
+
+	"code.google.com/p/go.text/transform"
+)
+
+// Transform implements the transform.Transformer interface. It may need to
+// write segments of up to MaxSegmentSize at once. Users should either catch
+// ErrShortDst and allow dst to grow or have dst be at least of size
+// MaxTransformChunkSize to be guaranteed of progress.
+func (f Form) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	n := 0
+	// Cap the maximum number of src bytes to check.
+	b := src
+	eof := atEOF
+	if ns := len(dst); ns < len(b) {
+		err = transform.ErrShortDst
+		eof = false
+		b = b[:ns]
+	}
+	i, ok := formTable[f].quickSpan(inputBytes(b), n, len(b), eof)
+	n += copy(dst[n:], b[n:i])
+	if !ok {
+		nDst, nSrc, err = f.transform(dst[n:], src[n:], atEOF)
+		return nDst + n, nSrc + n, err
+	}
+	if n < len(src) && !atEOF {
+		err = transform.ErrShortSrc
+	}
+	return n, n, err
+}
+
+func flushTransform(rb *reorderBuffer) bool {
+	// Write out (must fully fit in dst, or else it is a ErrShortDst).
+	if len(rb.out) < rb.nrune*utf8.UTFMax {
+		return false
+	}
+	rb.out = rb.out[rb.flushCopy(rb.out):]
+	return true
+}
+
+var errs = []error{nil, transform.ErrShortDst, transform.ErrShortSrc}
+
+// transform implements the transform.Transformer interface. It is only called
+// when quickSpan does not pass for a given string.
+func (f Form) transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	// TODO: get rid of reorderBuffer. See CL 23460044.
+	rb := reorderBuffer{}
+	rb.init(f, src)
+	for {
+		// Load segment into reorder buffer.
+		rb.setFlusher(dst[nDst:], flushTransform)
+		end := decomposeSegment(&rb, nSrc, atEOF)
+		if end < 0 {
+			return nDst, nSrc, errs[-end]
+		}
+		nDst = len(dst) - len(rb.out)
+		nSrc = end
+
+		// Next quickSpan.
+		end = rb.nsrc
+		eof := atEOF
+		if n := nSrc + len(dst) - nDst; n < end {
+			err = transform.ErrShortDst
+			end = n
+			eof = false
+		}
+		end, ok := rb.f.quickSpan(rb.src, nSrc, end, eof)
+		n := copy(dst[nDst:], rb.src.bytes[nSrc:end])
+		nSrc += n
+		nDst += n
+		if ok {
+			if n < rb.nsrc && !atEOF {
+				err = transform.ErrShortSrc
+			}
+			return nDst, nSrc, err
+		}
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/transform_test.go

@@ -0,0 +1,101 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import (
+	"fmt"
+	"testing"
+
+	"code.google.com/p/go.text/transform"
+)
+
+func TestTransform(t *testing.T) {
+	tests := []struct {
+		f       Form
+		in, out string
+		eof     bool
+		dstSize int
+		err     error
+	}{
+		{NFC, "ab", "ab", true, 2, nil},
+		{NFC, "qx", "qx", true, 2, nil},
+		{NFD, "qx", "qx", true, 2, nil},
+		{NFC, "", "", true, 1, nil},
+		{NFD, "", "", true, 1, nil},
+		{NFC, "", "", false, 1, nil},
+		{NFD, "", "", false, 1, nil},
+
+		// Normalized segment does not fit in destination.
+		{NFD, "ö", "", true, 1, transform.ErrShortDst},
+		{NFD, "ö", "", true, 2, transform.ErrShortDst},
+
+		// As an artifact of the algorithm, only full segments are written.
+		// This is not strictly required, and some bytes could be written.
+		// In practice, for Transform to not block, the destination buffer
+		// should be at least MaxSegmentSize to work anyway and these edge
+		// conditions will be relatively rare.
+		{NFC, "ab", "", true, 1, transform.ErrShortDst},
+		// This is even true for inert runes.
+		{NFC, "qx", "", true, 1, transform.ErrShortDst},
+		{NFC, "a\u0300abc", "\u00e0a", true, 4, transform.ErrShortDst},
+
+		// We cannot write a segment if succesive runes could still change the result.
+		{NFD, "ö", "", false, 3, transform.ErrShortSrc},
+		{NFC, "a\u0300", "", false, 4, transform.ErrShortSrc},
+		{NFD, "a\u0300", "", false, 4, transform.ErrShortSrc},
+		{NFC, "ö", "", false, 3, transform.ErrShortSrc},
+
+		{NFC, "a\u0300", "", true, 1, transform.ErrShortDst},
+		// Theoretically could fit, but won't due to simplified checks.
+		{NFC, "a\u0300", "", true, 2, transform.ErrShortDst},
+		{NFC, "a\u0300", "", true, 3, transform.ErrShortDst},
+		{NFC, "a\u0300", "\u00e0", true, 4, nil},
+
+		{NFD, "öa\u0300", "o\u0308", false, 8, transform.ErrShortSrc},
+		{NFD, "öa\u0300ö", "o\u0308a\u0300", true, 8, transform.ErrShortDst},
+		{NFD, "öa\u0300ö", "o\u0308a\u0300", false, 12, transform.ErrShortSrc},
+
+		// Illegal input is copied verbatim.
+		{NFD, "\xbd\xb2=\xbc ", "\xbd\xb2=\xbc ", true, 8, nil},
+	}
+	b := make([]byte, 100)
+	for i, tt := range tests {
+		nDst, _, err := tt.f.Transform(b[:tt.dstSize], []byte(tt.in), tt.eof)
+		out := string(b[:nDst])
+		if out != tt.out || err != tt.err {
+			t.Errorf("%d: was %+q (%v); want %+q (%v)", i, out, err, tt.out, tt.err)
+		}
+		if want := tt.f.String(tt.in)[:nDst]; want != out {
+			t.Errorf("%d: incorect normalization: was %+q; want %+q", i, out, want)
+		}
+	}
+}
+
+var transBufSizes = []int{
+	MaxTransformChunkSize,
+	3 * MaxTransformChunkSize / 2,
+	2 * MaxTransformChunkSize,
+	3 * MaxTransformChunkSize,
+	100 * MaxTransformChunkSize,
+}
+
+func doTransNorm(f Form, buf []byte, b []byte) []byte {
+	acc := []byte{}
+	for p := 0; p < len(b); {
+		nd, ns, _ := f.Transform(buf[:], b[p:], true)
+		p += ns
+		acc = append(acc, buf[:nd]...)
+	}
+	return acc
+}
+
+func TestTransformNorm(t *testing.T) {
+	for _, sz := range transBufSizes {
+		buf := make([]byte, sz)
+		runNormTests(t, fmt.Sprintf("Transform:%d", sz), func(f Form, out []byte, s string) []byte {
+			return doTransNorm(f, buf, append(out, s...))
+		})
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/trie.go

@@ -0,0 +1,232 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+type valueRange struct {
+	value  uint16 // header: value:stride
+	lo, hi byte   // header: lo:n
+}
+
+type trie struct {
+	index        []uint8
+	values       []uint16
+	sparse       []valueRange
+	sparseOffset []uint16
+	cutoff       uint8 // indices >= cutoff are sparse
+}
+
+// lookupValue determines the type of block n and looks up the value for b.
+// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
+// is a list of ranges with an accompanying value. Given a matching range r,
+// the value for b is by r.value + (b - r.lo) * stride.
+func (t *trie) lookupValue(n uint8, b byte) uint16 {
+	if n < t.cutoff {
+		return t.values[uint16(n)<<6+uint16(b)]
+	}
+	offset := t.sparseOffset[n-t.cutoff]
+	header := t.sparse[offset]
+	lo := offset + 1
+	hi := lo + uint16(header.lo)
+	for lo < hi {
+		m := lo + (hi-lo)/2
+		r := t.sparse[m]
+		if r.lo <= b && b <= r.hi {
+			return r.value + uint16(b-r.lo)*header.value
+		}
+		if b < r.lo {
+			hi = m
+		} else {
+			lo = m + 1
+		}
+	}
+	return 0
+}
+
+const (
+	t1 = 0x00 // 0000 0000
+	tx = 0x80 // 1000 0000
+	t2 = 0xC0 // 1100 0000
+	t3 = 0xE0 // 1110 0000
+	t4 = 0xF0 // 1111 0000
+	t5 = 0xF8 // 1111 1000
+	t6 = 0xFC // 1111 1100
+	te = 0xFE // 1111 1110
+)
+
+// lookup returns the trie value for the first UTF-8 encoding in s and
+// the width in bytes of this encoding. The size will be 0 if s does not
+// hold enough bytes to complete the encoding. len(s) must be greater than 0.
+func (t *trie) lookup(s []byte) (v uint16, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < tx:
+		return t.values[c0], 1
+	case c0 < t2:
+		return 0, 1
+	case c0 < t3:
+		if len(s) < 2 {
+			return 0, 0
+		}
+		i := t.index[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		return t.lookupValue(i, c1), 2
+	case c0 < t4:
+		if len(s) < 3 {
+			return 0, 0
+		}
+		i := t.index[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		o := uint16(i)<<6 + uint16(c1)
+		i = t.index[o]
+		c2 := s[2]
+		if c2 < tx || t2 <= c2 {
+			return 0, 2
+		}
+		return t.lookupValue(i, c2), 3
+	case c0 < t5:
+		if len(s) < 4 {
+			return 0, 0
+		}
+		i := t.index[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		o := uint16(i)<<6 + uint16(c1)
+		i = t.index[o]
+		c2 := s[2]
+		if c2 < tx || t2 <= c2 {
+			return 0, 2
+		}
+		o = uint16(i)<<6 + uint16(c2)
+		i = t.index[o]
+		c3 := s[3]
+		if c3 < tx || t2 <= c3 {
+			return 0, 3
+		}
+		return t.lookupValue(i, c3), 4
+	}
+	// Illegal rune
+	return 0, 1
+}
+
+// lookupString returns the trie value for the first UTF-8 encoding in s and
+// the width in bytes of this encoding. The size will be 0 if s does not
+// hold enough bytes to complete the encoding. len(s) must be greater than 0.
+func (t *trie) lookupString(s string) (v uint16, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < tx:
+		return t.values[c0], 1
+	case c0 < t2:
+		return 0, 1
+	case c0 < t3:
+		if len(s) < 2 {
+			return 0, 0
+		}
+		i := t.index[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		return t.lookupValue(i, c1), 2
+	case c0 < t4:
+		if len(s) < 3 {
+			return 0, 0
+		}
+		i := t.index[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		o := uint16(i)<<6 + uint16(c1)
+		i = t.index[o]
+		c2 := s[2]
+		if c2 < tx || t2 <= c2 {
+			return 0, 2
+		}
+		return t.lookupValue(i, c2), 3
+	case c0 < t5:
+		if len(s) < 4 {
+			return 0, 0
+		}
+		i := t.index[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		o := uint16(i)<<6 + uint16(c1)
+		i = t.index[o]
+		c2 := s[2]
+		if c2 < tx || t2 <= c2 {
+			return 0, 2
+		}
+		o = uint16(i)<<6 + uint16(c2)
+		i = t.index[o]
+		c3 := s[3]
+		if c3 < tx || t2 <= c3 {
+			return 0, 3
+		}
+		return t.lookupValue(i, c3), 4
+	}
+	// Illegal rune
+	return 0, 1
+}
+
+// lookupUnsafe returns the trie value for the first UTF-8 encoding in s.
+// s must hold a full encoding.
+func (t *trie) lookupUnsafe(s []byte) uint16 {
+	c0 := s[0]
+	if c0 < tx {
+		return t.values[c0]
+	}
+	if c0 < t2 {
+		return 0
+	}
+	i := t.index[c0]
+	if c0 < t3 {
+		return t.lookupValue(i, s[1])
+	}
+	i = t.index[uint16(i)<<6+uint16(s[1])]
+	if c0 < t4 {
+		return t.lookupValue(i, s[2])
+	}
+	i = t.index[uint16(i)<<6+uint16(s[2])]
+	if c0 < t5 {
+		return t.lookupValue(i, s[3])
+	}
+	return 0
+}
+
+// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s.
+// s must hold a full encoding.
+func (t *trie) lookupStringUnsafe(s string) uint16 {
+	c0 := s[0]
+	if c0 < tx {
+		return t.values[c0]
+	}
+	if c0 < t2 {
+		return 0
+	}
+	i := t.index[c0]
+	if c0 < t3 {
+		return t.lookupValue(i, s[1])
+	}
+	i = t.index[uint16(i)<<6+uint16(s[1])]
+	if c0 < t4 {
+		return t.lookupValue(i, s[2])
+	}
+	i = t.index[uint16(i)<<6+uint16(s[2])]
+	if c0 < t5 {
+		return t.lookupValue(i, s[3])
+	}
+	return 0
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/trie_test.go

@@ -0,0 +1,152 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import (
+	"testing"
+	"unicode/utf8"
+)
+
+// Test data is located in triedata_test.go; generated by maketesttables.
+var testdata = testdataTrie
+
+type rangeTest struct {
+	block   uint8
+	lookup  byte
+	result  uint16
+	table   []valueRange
+	offsets []uint16
+}
+
+var range1Off = []uint16{0, 2}
+var range1 = []valueRange{
+	{0, 1, 0},
+	{1, 0x80, 0x80},
+	{0, 2, 0},
+	{1, 0x80, 0x80},
+	{9, 0xff, 0xff},
+}
+
+var rangeTests = []rangeTest{
+	{10, 0x80, 1, range1, range1Off},
+	{10, 0x00, 0, range1, range1Off},
+	{11, 0x80, 1, range1, range1Off},
+	{11, 0xff, 9, range1, range1Off},
+	{11, 0x00, 0, range1, range1Off},
+}
+
+func TestLookupSparse(t *testing.T) {
+	for i, test := range rangeTests {
+		n := trie{sparse: test.table, sparseOffset: test.offsets, cutoff: 10}
+		v := n.lookupValue(test.block, test.lookup)
+		if v != test.result {
+			t.Errorf("LookupSparse:%d: found %X; want %X", i, v, test.result)
+		}
+	}
+}
+
+// Test cases for illegal runes.
+type trietest struct {
+	size  int
+	bytes []byte
+}
+
+var tests = []trietest{
+	// illegal runes
+	{1, []byte{0x80}},
+	{1, []byte{0xFF}},
+	{1, []byte{t2, tx - 1}},
+	{1, []byte{t2, t2}},
+	{2, []byte{t3, tx, tx - 1}},
+	{2, []byte{t3, tx, t2}},
+	{1, []byte{t3, tx - 1, tx}},
+	{3, []byte{t4, tx, tx, tx - 1}},
+	{3, []byte{t4, tx, tx, t2}},
+	{1, []byte{t4, t2, tx, tx - 1}},
+	{2, []byte{t4, tx, t2, tx - 1}},
+
+	// short runes
+	{0, []byte{t2}},
+	{0, []byte{t3, tx}},
+	{0, []byte{t4, tx, tx}},
+
+	// we only support UTF-8 up to utf8.UTFMax bytes (4 bytes)
+	{1, []byte{t5, tx, tx, tx, tx}},
+	{1, []byte{t6, tx, tx, tx, tx, tx}},
+}
+
+func mkUTF8(r rune) ([]byte, int) {
+	var b [utf8.UTFMax]byte
+	sz := utf8.EncodeRune(b[:], r)
+	return b[:sz], sz
+}
+
+func TestLookup(t *testing.T) {
+	for i, tt := range testRunes {
+		b, szg := mkUTF8(tt)
+		v, szt := testdata.lookup(b)
+		if int(v) != i {
+			t.Errorf("lookup(%U): found value %#x, expected %#x", tt, v, i)
+		}
+		if szt != szg {
+			t.Errorf("lookup(%U): found size %d, expected %d", tt, szt, szg)
+		}
+	}
+	for i, tt := range tests {
+		v, sz := testdata.lookup(tt.bytes)
+		if v != 0 {
+			t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
+		}
+		if sz != tt.size {
+			t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
+		}
+	}
+	// Verify defaults.
+	if v, _ := testdata.lookup([]byte{0xC1, 0x8C}); v != 0 {
+		t.Errorf("lookup of non-existing rune should be 0; found %X", v)
+	}
+}
+
+func TestLookupUnsafe(t *testing.T) {
+	for i, tt := range testRunes {
+		b, _ := mkUTF8(tt)
+		v := testdata.lookupUnsafe(b)
+		if int(v) != i {
+			t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i)
+		}
+	}
+}
+
+func TestLookupString(t *testing.T) {
+	for i, tt := range testRunes {
+		b, szg := mkUTF8(tt)
+		v, szt := testdata.lookupString(string(b))
+		if int(v) != i {
+			t.Errorf("lookup(%U): found value %#x, expected %#x", i, v, i)
+		}
+		if szt != szg {
+			t.Errorf("lookup(%U): found size %d, expected %d", i, szt, szg)
+		}
+	}
+	for i, tt := range tests {
+		v, sz := testdata.lookupString(string(tt.bytes))
+		if int(v) != 0 {
+			t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
+		}
+		if sz != tt.size {
+			t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
+		}
+	}
+}
+
+func TestLookupStringUnsafe(t *testing.T) {
+	for i, tt := range testRunes {
+		b, _ := mkUTF8(tt)
+		v := testdata.lookupStringUnsafe(string(b))
+		if int(v) != i {
+			t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i)
+		}
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/triedata_test.go

@@ -0,0 +1,85 @@
+// Generated by running
+//	maketesttables
+// DO NOT EDIT
+
+package norm
+
+var testRunes = []int32{1, 12, 127, 128, 256, 2047, 2048, 2457, 65535, 65536, 65793, 1114111, 512, 513, 514, 528, 533}
+
+// testdataValues: 192 entries, 384 bytes
+// Block 2 is the null block.
+var testdataValues = [192]uint16{
+	// Block 0x0, offset 0x0
+	0x000c: 0x0001,
+	// Block 0x1, offset 0x40
+	0x007f: 0x0002,
+	// Block 0x2, offset 0x80
+}
+
+// testdataSparseOffset: 10 entries, 20 bytes
+var testdataSparseOffset = []uint16{0x0, 0x2, 0x4, 0x8, 0xa, 0xc, 0xe, 0x10, 0x12, 0x14}
+
+// testdataSparseValues: 22 entries, 88 bytes
+var testdataSparseValues = [22]valueRange{
+	// Block 0x0, offset 0x1
+	{value: 0x0000, lo: 0x01},
+	{value: 0x0003, lo: 0x80, hi: 0x80},
+	// Block 0x1, offset 0x2
+	{value: 0x0000, lo: 0x01},
+	{value: 0x0004, lo: 0x80, hi: 0x80},
+	// Block 0x2, offset 0x3
+	{value: 0x0001, lo: 0x03},
+	{value: 0x000c, lo: 0x80, hi: 0x82},
+	{value: 0x000f, lo: 0x90, hi: 0x90},
+	{value: 0x0010, lo: 0x95, hi: 0x95},
+	// Block 0x3, offset 0x4
+	{value: 0x0000, lo: 0x01},
+	{value: 0x0005, lo: 0xbf, hi: 0xbf},
+	// Block 0x4, offset 0x5
+	{value: 0x0000, lo: 0x01},
+	{value: 0x0006, lo: 0x80, hi: 0x80},
+	// Block 0x5, offset 0x6
+	{value: 0x0000, lo: 0x01},
+	{value: 0x0007, lo: 0x99, hi: 0x99},
+	// Block 0x6, offset 0x7
+	{value: 0x0000, lo: 0x01},
+	{value: 0x0008, lo: 0xbf, hi: 0xbf},
+	// Block 0x7, offset 0x8
+	{value: 0x0000, lo: 0x01},
+	{value: 0x0009, lo: 0x80, hi: 0x80},
+	// Block 0x8, offset 0x9
+	{value: 0x0000, lo: 0x01},
+	{value: 0x000a, lo: 0x81, hi: 0x81},
+	// Block 0x9, offset 0xa
+	{value: 0x0000, lo: 0x01},
+	{value: 0x000b, lo: 0xbf, hi: 0xbf},
+}
+
+// testdataLookup: 640 bytes
+// Block 0 is the null block.
+var testdataLookup = [640]uint8{
+	// Block 0x0, offset 0x0
+	// Block 0x1, offset 0x40
+	// Block 0x2, offset 0x80
+	// Block 0x3, offset 0xc0
+	0x0c2: 0x01, 0x0c4: 0x02,
+	0x0c8: 0x03,
+	0x0df: 0x04,
+	0x0e0: 0x02,
+	0x0ef: 0x03,
+	0x0f0: 0x05, 0x0f4: 0x07,
+	// Block 0x4, offset 0x100
+	0x120: 0x05, 0x126: 0x06,
+	// Block 0x5, offset 0x140
+	0x17f: 0x07,
+	// Block 0x6, offset 0x180
+	0x180: 0x08, 0x184: 0x09,
+	// Block 0x7, offset 0x1c0
+	0x1d0: 0x04,
+	// Block 0x8, offset 0x200
+	0x23f: 0x0a,
+	// Block 0x9, offset 0x240
+	0x24f: 0x06,
+}
+
+var testdataTrie = trie{testdataLookup[:], testdataValues[:], testdataSparseValues[:], testdataSparseOffset[:], 1}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/triegen.go

@@ -0,0 +1,317 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Trie table generator.
+// Used by make*tables tools to generate a go file with trie data structures
+// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte
+// sequence are used to lookup offsets in the index table to be used for the
+// next byte. The last byte is used to index into a table with 16-bit values.
+
+package main
+
+import (
+	"fmt"
+	"hash/crc32"
+	"log"
+	"unicode/utf8"
+)
+
+const (
+	blockSize        = 64
+	blockOffset      = 2 // Subtract two blocks to compensate for the 0x80 added to continuation bytes.
+	maxSparseEntries = 16
+)
+
+// Intermediate trie structure
+type trieNode struct {
+	table [256]*trieNode
+	value int
+	b     byte
+	leaf  bool
+}
+
+func newNode() *trieNode {
+	return new(trieNode)
+}
+
+func (n trieNode) String() string {
+	s := fmt.Sprint("trieNode{table: { non-nil at index: ")
+	for i, v := range n.table {
+		if v != nil {
+			s += fmt.Sprintf("%d, ", i)
+		}
+	}
+	s += fmt.Sprintf("}, value:%#x, b:%#x leaf:%v}", n.value, n.b, n.leaf)
+	return s
+}
+
+func (n trieNode) isInternal() bool {
+	internal := true
+	for i := 0; i < 256; i++ {
+		if nn := n.table[i]; nn != nil {
+			if !internal && !nn.leaf {
+				log.Fatalf("triegen: isInternal: node contains both leaf and non-leaf children (%v)", n)
+			}
+			internal = internal && !nn.leaf
+		}
+	}
+	return internal
+}
+
+func (n trieNode) mostFrequentStride() int {
+	counts := make(map[int]int)
+	v := 0
+	for _, t := range n.table[0x80 : 0x80+blockSize] {
+		if t != nil {
+			if stride := t.value - v; v != 0 && stride >= 0 {
+				counts[stride]++
+			}
+			v = t.value
+		} else {
+			v = 0
+		}
+	}
+	var maxs, maxc int
+	for stride, cnt := range counts {
+		if cnt > maxc || (cnt == maxc && stride < maxs) {
+			maxs, maxc = stride, cnt
+		}
+	}
+	return maxs
+}
+
+func (n trieNode) countSparseEntries() int {
+	stride := n.mostFrequentStride()
+	var count, v int
+	for _, t := range n.table[0x80 : 0x80+blockSize] {
+		tv := 0
+		if t != nil {
+			tv = t.value
+		}
+		if tv-v != stride {
+			if tv != 0 {
+				count++
+			}
+		}
+		v = tv
+	}
+	return count
+}
+
+func (n *trieNode) insert(r rune, value uint16) {
+	var p [utf8.UTFMax]byte
+	sz := utf8.EncodeRune(p[:], r)
+
+	for i := 0; i < sz; i++ {
+		if n.leaf {
+			log.Fatalf("triegen: insert: node (%#v) should not be a leaf", n)
+		}
+		nn := n.table[p[i]]
+		if nn == nil {
+			nn = newNode()
+			nn.b = p[i]
+			n.table[p[i]] = nn
+		}
+		n = nn
+	}
+	n.value = int(value)
+	n.leaf = true
+}
+
+type nodeIndex struct {
+	lookupBlocks []*trieNode
+	valueBlocks  []*trieNode
+	sparseBlocks []*trieNode
+	sparseOffset []uint16
+	sparseCount  int
+
+	lookupBlockIdx map[uint32]int
+	valueBlockIdx  map[uint32]int
+}
+
+func newIndex() *nodeIndex {
+	index := &nodeIndex{}
+	index.lookupBlocks = make([]*trieNode, 0)
+	index.valueBlocks = make([]*trieNode, 0)
+	index.sparseBlocks = make([]*trieNode, 0)
+	index.sparseOffset = make([]uint16, 1)
+	index.lookupBlockIdx = make(map[uint32]int)
+	index.valueBlockIdx = make(map[uint32]int)
+	return index
+}
+
+func computeOffsets(index *nodeIndex, n *trieNode) int {
+	if n.leaf {
+		return n.value
+	}
+	hasher := crc32.New(crc32.MakeTable(crc32.IEEE))
+	// We only index continuation bytes.
+	for i := 0; i < blockSize; i++ {
+		v := 0
+		if nn := n.table[0x80+i]; nn != nil {
+			v = computeOffsets(index, nn)
+		}
+		hasher.Write([]byte{uint8(v >> 8), uint8(v)})
+	}
+	h := hasher.Sum32()
+	if n.isInternal() {
+		v, ok := index.lookupBlockIdx[h]
+		if !ok {
+			v = len(index.lookupBlocks) - blockOffset
+			index.lookupBlocks = append(index.lookupBlocks, n)
+			index.lookupBlockIdx[h] = v
+		}
+		n.value = v
+	} else {
+		v, ok := index.valueBlockIdx[h]
+		if !ok {
+			if c := n.countSparseEntries(); c > maxSparseEntries {
+				v = len(index.valueBlocks) - blockOffset
+				index.valueBlocks = append(index.valueBlocks, n)
+				index.valueBlockIdx[h] = v
+			} else {
+				v = -len(index.sparseOffset)
+				index.sparseBlocks = append(index.sparseBlocks, n)
+				index.sparseOffset = append(index.sparseOffset, uint16(index.sparseCount))
+				index.sparseCount += c + 1
+				index.valueBlockIdx[h] = v
+			}
+		}
+		n.value = v
+	}
+	return n.value
+}
+
+func printValueBlock(nr int, n *trieNode, offset int) {
+	boff := nr * blockSize
+	fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
+	var printnewline bool
+	for i := 0; i < blockSize; i++ {
+		if i%6 == 0 {
+			printnewline = true
+		}
+		v := 0
+		if nn := n.table[i+offset]; nn != nil {
+			v = nn.value
+		}
+		if v != 0 {
+			if printnewline {
+				fmt.Printf("\n")
+				printnewline = false
+			}
+			fmt.Printf("%#04x:%#04x, ", boff+i, v)
+		}
+	}
+}
+
+func printSparseBlock(nr int, n *trieNode) {
+	boff := -n.value
+	fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
+	v := 0
+	//stride := f(n)
+	stride := n.mostFrequentStride()
+	c := n.countSparseEntries()
+	fmt.Printf("\n{value:%#04x,lo:%#02x},", stride, uint8(c))
+	for i, nn := range n.table[0x80 : 0x80+blockSize] {
+		nv := 0
+		if nn != nil {
+			nv = nn.value
+		}
+		if nv-v != stride {
+			if v != 0 {
+				fmt.Printf(",hi:%#02x},", 0x80+i-1)
+			}
+			if nv != 0 {
+				fmt.Printf("\n{value:%#04x,lo:%#02x", nv, nn.b)
+			}
+		}
+		v = nv
+	}
+	if v != 0 {
+		fmt.Printf(",hi:%#02x},", 0x80+blockSize-1)
+	}
+}
+
+func printLookupBlock(nr int, n *trieNode, offset, cutoff int) {
+	boff := nr * blockSize
+	fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
+	var printnewline bool
+	for i := 0; i < blockSize; i++ {
+		if i%8 == 0 {
+			printnewline = true
+		}
+		v := 0
+		if nn := n.table[i+offset]; nn != nil {
+			v = nn.value
+		}
+		if v != 0 {
+			if v < 0 {
+				v = -v - 1 + cutoff
+			}
+			if printnewline {
+				fmt.Printf("\n")
+				printnewline = false
+			}
+			fmt.Printf("%#03x:%#02x, ", boff+i, v)
+		}
+	}
+}
+
+// printTables returns the size in bytes of the generated tables.
+func (t *trieNode) printTables(name string) int {
+	index := newIndex()
+	// Values for 7-bit ASCII are stored in first two block, followed by nil block.
+	index.valueBlocks = append(index.valueBlocks, nil, nil, nil)
+	// First byte of multi-byte UTF-8 codepoints are indexed in 4th block.
+	index.lookupBlocks = append(index.lookupBlocks, nil, nil, nil, nil)
+	// Index starter bytes of multi-byte UTF-8.
+	for i := 0xC0; i < 0x100; i++ {
+		if t.table[i] != nil {
+			computeOffsets(index, t.table[i])
+		}
+	}
+
+	nv := len(index.valueBlocks) * blockSize
+	fmt.Printf("// %sValues: %d entries, %d bytes\n", name, nv, nv*2)
+	fmt.Printf("// Block 2 is the null block.\n")
+	fmt.Printf("var %sValues = [%d]uint16 {", name, nv)
+	printValueBlock(0, t, 0)
+	printValueBlock(1, t, 64)
+	printValueBlock(2, newNode(), 0)
+	for i := 3; i < len(index.valueBlocks); i++ {
+		printValueBlock(i, index.valueBlocks[i], 0x80)
+	}
+	fmt.Print("\n}\n\n")
+
+	ls := len(index.sparseBlocks)
+	fmt.Printf("// %sSparseOffset: %d entries, %d bytes\n", name, ls, ls*2)
+	fmt.Printf("var %sSparseOffset = %#v\n\n", name, index.sparseOffset[1:])
+
+	ns := index.sparseCount
+	fmt.Printf("// %sSparseValues: %d entries, %d bytes\n", name, ns, ns*4)
+	fmt.Printf("var %sSparseValues = [%d]valueRange {", name, ns)
+	for i, n := range index.sparseBlocks {
+		printSparseBlock(i, n)
+	}
+	fmt.Print("\n}\n\n")
+
+	cutoff := len(index.valueBlocks) - blockOffset
+	ni := len(index.lookupBlocks) * blockSize
+	fmt.Printf("// %sLookup: %d bytes\n", name, ni)
+	fmt.Printf("// Block 0 is the null block.\n")
+	fmt.Printf("var %sLookup = [%d]uint8 {", name, ni)
+	printLookupBlock(0, newNode(), 0, cutoff)
+	printLookupBlock(1, newNode(), 0, cutoff)
+	printLookupBlock(2, newNode(), 0, cutoff)
+	printLookupBlock(3, t, 0xC0, cutoff)
+	for i := 4; i < len(index.lookupBlocks); i++ {
+		printLookupBlock(i, index.lookupBlocks[i], 0x80, cutoff)
+	}
+	fmt.Print("\n}\n\n")
+	fmt.Printf("var %sTrie = trie{ %sLookup[:], %sValues[:], %sSparseValues[:], %sSparseOffset[:], %d}\n\n",
+		name, name, name, name, name, cutoff)
+	return nv*2 + ns*4 + ni + ls*2
+}

Godeps/_workspace/src/github.com/calmh/ini/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2013 Jakob Borg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+- The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Godeps/_workspace/src/github.com/calmh/ini/README.md

@@ -0,0 +1,39 @@
+ini [![Build Status](https://drone.io/github.com/calmh/ini/status.png)](https://drone.io/github.com/calmh/ini/latest)
+===
+
+Yet another .INI file parser / writer. Created because the existing ones
+were either not general enough (allowing easy access to all parts of the
+original file) or made annoying assumptions about the format. And
+probably equal parts NIH. You might want to just write your own instead
+of using this one, you know that's where you'll end up in the end
+anyhow.
+
+Documentation
+-------------
+
+http://godoc.org/github.com/calmh/ini
+
+Example
+-------
+
+```go
+fd, _ := os.Open("foo.ini")
+cfg := ini.Parse(fd)
+fd.Close()
+
+val := cfg.Get("general", "foo")
+cfg.Set("general", "bar", "baz")
+
+fd, _ = os.Create("bar.ini")
+err := cfg.Write(fd)
+if err != nil {
+	// ...
+}
+err = fd.Close()
+
+```
+
+License
+-------
+
+MIT

Godeps/_workspace/src/github.com/calmh/ini/ini.go

@@ -0,0 +1,235 @@
+// Package ini provides trivial parsing of .INI format files.
+package ini
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// Config is a parsed INI format file.
+type Config struct {
+	sections []section
+	comments []string
+}
+
+type section struct {
+	name     string
+	comments []string
+	options  []option
+}
+
+type option struct {
+	name, value string
+}
+
+var (
+	iniSectionRe = regexp.MustCompile(`^\[(.+)\]$`)
+	iniOptionRe  = regexp.MustCompile(`^([^\s=]+)\s*=\s*(.+?)$`)
+)
+
+// Sections returns the list of sections in the file.
+func (c *Config) Sections() []string {
+	var sections []string
+	for _, sect := range c.sections {
+		sections = append(sections, sect.name)
+	}
+	return sections
+}
+
+// Options returns the list of options in a given section.
+func (c *Config) Options(section string) []string {
+	var options []string
+	for _, sect := range c.sections {
+		if sect.name == section {
+			for _, opt := range sect.options {
+				options = append(options, opt.name)
+			}
+			break
+		}
+	}
+	return options
+}
+
+// OptionMap returns the map option => value for a given section.
+func (c *Config) OptionMap(section string) map[string]string {
+	options := make(map[string]string)
+	for _, sect := range c.sections {
+		if sect.name == section {
+			for _, opt := range sect.options {
+				options[opt.name] = opt.value
+			}
+			break
+		}
+	}
+	return options
+}
+
+// Comments returns the list of comments in a given section.
+// For the empty string, returns the file comments.
+func (c *Config) Comments(section string) []string {
+	if section == "" {
+		return c.comments
+	}
+	for _, sect := range c.sections {
+		if sect.name == section {
+			return sect.comments
+		}
+	}
+	return nil
+}
+
+// AddComments appends the comment to the list of comments for the section.
+func (c *Config) AddComment(sect, comment string) {
+	if sect == "" {
+		c.comments = append(c.comments, comment)
+		return
+	}
+
+	for i, s := range c.sections {
+		if s.name == sect {
+			c.sections[i].comments = append(s.comments, comment)
+			return
+		}
+	}
+
+	c.sections = append(c.sections, section{
+		name:     sect,
+		comments: []string{comment},
+	})
+}
+
+// Parse reads the given io.Reader and returns a parsed Config object.
+func Parse(stream io.Reader) Config {
+	var cfg Config
+	var curSection string
+
+	scanner := bufio.NewScanner(bufio.NewReader(stream))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") {
+			comment := strings.TrimLeft(line, ";# ")
+			cfg.AddComment(curSection, comment)
+		} else if len(line) > 0 {
+			if m := iniSectionRe.FindStringSubmatch(line); len(m) > 0 {
+				curSection = m[1]
+			} else if m := iniOptionRe.FindStringSubmatch(line); len(m) > 0 {
+				key := m[1]
+				val := m[2]
+				if !strings.Contains(val, "\"") {
+					// If val does not contain any quote characers, we can make it
+					// a quoted string and safely let strconv.Unquote sort out any
+					// escapes
+					val = "\"" + val + "\""
+				}
+				if val[0] == '"' {
+					val, _ = strconv.Unquote(val)
+				}
+
+				cfg.Set(curSection, key, val)
+			}
+		}
+	}
+	return cfg
+}
+
+// Write writes the sections and options to the io.Writer in INI format.
+func (c *Config) Write(out io.Writer) error {
+	for _, cmt := range c.comments {
+		fmt.Fprintln(out, "; "+cmt)
+	}
+	if len(c.comments) > 0 {
+		fmt.Fprintln(out)
+	}
+
+	for _, sect := range c.sections {
+		fmt.Fprintf(out, "[%s]\n", sect.name)
+		for _, cmt := range sect.comments {
+			fmt.Fprintln(out, "; "+cmt)
+		}
+		for _, opt := range sect.options {
+			val := opt.value
+			if len(val) == 0 {
+				continue
+			}
+
+			// Quote the string if it begins or ends with space
+			needsQuoting := val[0] == ' ' || val[len(val)-1] == ' '
+
+			if !needsQuoting {
+				// Quote the string if it contains any unprintable characters
+				for _, r := range val {
+					if !strconv.IsPrint(r) {
+						needsQuoting = true
+						break
+					}
+				}
+			}
+
+			if needsQuoting {
+				val = strconv.Quote(val)
+			}
+
+			fmt.Fprintf(out, "%s=%s\n", opt.name, val)
+		}
+		fmt.Fprintln(out)
+	}
+	return nil
+}
+
+// Get gets the value from the specified section and key name, or the empty
+// string if either the section or the key is missing.
+func (c *Config) Get(section, key string) string {
+	for _, sect := range c.sections {
+		if sect.name == section {
+			for _, opt := range sect.options {
+				if opt.name == key {
+					return opt.value
+				}
+			}
+			return ""
+		}
+	}
+	return ""
+}
+
+// Set sets a value for an option in a section. If the option exists, it's
+// value will be overwritten. If the option does not exist, it will be added.
+// If the section does not exist, it will be added and the option added to it.
+func (c *Config) Set(sectionName, key, value string) {
+	for i, sect := range c.sections {
+		if sect.name == sectionName {
+			for j, opt := range sect.options {
+				if opt.name == key {
+					c.sections[i].options[j].value = value
+					return
+				}
+			}
+			c.sections[i].options = append(sect.options, option{key, value})
+			return
+		}
+	}
+
+	c.sections = append(c.sections, section{
+		name:    sectionName,
+		options: []option{{key, value}},
+	})
+}
+
+// Delete removes the option from the specified section.
+func (c *Config) Delete(section, key string) {
+	for sn, sect := range c.sections {
+		if sect.name == section {
+			for i, opt := range sect.options {
+				if opt.name == key {
+					c.sections[sn].options = append(sect.options[:i], sect.options[i+1:]...)
+					return
+				}
+			}
+			return
+		}
+	}
+}

Godeps/_workspace/src/github.com/calmh/ini/ini_test.go

@@ -0,0 +1,214 @@
+package ini_test
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/calmh/ini"
+)
+
+func TestParseValues(t *testing.T) {
+	strs := []string{
+		`[general]`,
+		`k1=v1`,
+		`k2 = v2`,
+		` k3 = v3 `,
+		`k4=" quoted spaces "`,
+		`k5 = " quoted spaces " `,
+		`k6 = with\nnewline`,
+		`k7 = "with\nnewline"`,
+		`k8 = a "quoted" word`,
+		`k9 = "a \"quoted\" word"`,
+	}
+	buf := bytes.NewBufferString(strings.Join(strs, "\n"))
+	cfg := ini.Parse(buf)
+
+	correct := map[string]string{
+		"k1": "v1",
+		"k2": "v2",
+		"k3": "v3",
+		"k4": " quoted spaces ",
+		"k5": " quoted spaces ",
+		"k6": "with\nnewline",
+		"k7": "with\nnewline",
+		"k8": "a \"quoted\" word",
+		"k9": "a \"quoted\" word",
+	}
+
+	for k, v := range correct {
+		if v2 := cfg.Get("general", k); v2 != v {
+			t.Errorf("Incorrect general.%s, %q != %q", k, v2, v)
+		}
+	}
+
+	if v := cfg.Get("general", "nonexistant"); v != "" {
+		t.Errorf("Unexpected non-empty value %q", v)
+	}
+}
+
+func TestParseComments(t *testing.T) {
+	strs := []string{
+		";file comment 1",   // No leading space
+		"; file comment 2 ", // Trailing space
+		";  file comment 3", // Multiple leading spaces
+		"[general]",
+		"; b general comment 1", // Comments in unsorted order
+		"somekey = somevalue",
+		"; a general comment 2",
+		"[other]",
+		"; other comment 1", // Comments in section with no values
+		"; other comment 2",
+		"[other2]",
+		"; other2 comment 1",
+		"; other2 comment 2", // Comments on last section
+		"somekey = somevalue",
+	}
+	buf := bytes.NewBufferString(strings.Join(strs, "\n"))
+
+	correct := map[string][]string{
+		"":        []string{"file comment 1", "file comment 2", "file comment 3"},
+		"general": []string{"b general comment 1", "a general comment 2"},
+		"other":   []string{"other comment 1", "other comment 2"},
+		"other2":  []string{"other2 comment 1", "other2 comment 2"},
+	}
+
+	cfg := ini.Parse(buf)
+
+	for section, comments := range correct {
+		cmts := cfg.Comments(section)
+		if len(cmts) != len(comments) {
+			t.Errorf("Incorrect number of comments for section %q: %d != %d", section, len(cmts), len(comments))
+		} else {
+			for i := range comments {
+				if cmts[i] != comments[i] {
+					t.Errorf("Incorrect comment: %q != %q", cmts[i], comments[i])
+				}
+			}
+		}
+	}
+}
+
+func TestWrite(t *testing.T) {
+	cfg := ini.Config{}
+	cfg.Set("general", "k1", "v1")
+	cfg.Set("general", "k2", "foo bar")
+	cfg.Set("general", "k3", " foo bar ")
+	cfg.Set("general", "k4", "foo\nbar")
+
+	var out bytes.Buffer
+	cfg.Write(&out)
+
+	correct := `[general]
+k1=v1
+k2=foo bar
+k3=" foo bar "
+k4="foo\nbar"
+
+`
+	if s := out.String(); s != correct {
+		t.Errorf("Incorrect written .INI:\n%s\ncorrect:\n%s", s, correct)
+	}
+}
+
+func TestSet(t *testing.T) {
+	buf := bytes.NewBufferString("[general]\nfoo=bar\nfoo2=bar2\n")
+	cfg := ini.Parse(buf)
+
+	cfg.Set("general", "foo", "baz")  // Overwrite existing
+	cfg.Set("general", "baz", "quux") // Create new value
+	cfg.Set("other", "baz2", "quux2") // Create new section + value
+
+	var out bytes.Buffer
+	cfg.Write(&out)
+
+	correct := `[general]
+foo=baz
+foo2=bar2
+baz=quux
+
+[other]
+baz2=quux2
+
+`
+
+	if s := out.String(); s != correct {
+		t.Errorf("Incorrect INI after set:\n%s", s)
+	}
+}
+
+func TestDelete(t *testing.T) {
+	buf := bytes.NewBufferString("[general]\nfoo=bar\nfoo2=bar2\nfoo3=baz\n")
+	cfg := ini.Parse(buf)
+	cfg.Delete("general", "foo")
+	out := new(bytes.Buffer)
+	cfg.Write(out)
+	correct := "[general]\nfoo2=bar2\nfoo3=baz\n\n"
+
+	if s := out.String(); s != correct {
+		t.Errorf("Incorrect INI after delete:\n%s", s)
+	}
+
+	buf = bytes.NewBufferString("[general]\nfoo=bar\nfoo2=bar2\nfoo3=baz\n")
+	cfg = ini.Parse(buf)
+	cfg.Delete("general", "foo2")
+	out = new(bytes.Buffer)
+	cfg.Write(out)
+	correct = "[general]\nfoo=bar\nfoo3=baz\n\n"
+
+	if s := out.String(); s != correct {
+		t.Errorf("Incorrect INI after delete:\n%s", s)
+	}
+
+	buf = bytes.NewBufferString("[general]\nfoo=bar\nfoo2=bar2\nfoo3=baz\n")
+	cfg = ini.Parse(buf)
+	cfg.Delete("general", "foo3")
+	out = new(bytes.Buffer)
+	cfg.Write(out)
+	correct = "[general]\nfoo=bar\nfoo2=bar2\n\n"
+
+	if s := out.String(); s != correct {
+		t.Errorf("Incorrect INI after delete:\n%s", s)
+	}
+}
+
+func TestSetManyEquals(t *testing.T) {
+	buf := bytes.NewBufferString("[general]\nfoo=bar==\nfoo2=bar2==\n")
+	cfg := ini.Parse(buf)
+
+	cfg.Set("general", "foo", "baz==")
+
+	var out bytes.Buffer
+	cfg.Write(&out)
+
+	correct := `[general]
+foo=baz==
+foo2=bar2==
+
+`
+
+	if s := out.String(); s != correct {
+		t.Errorf("Incorrect INI after set:\n%s", s)
+	}
+}
+
+func TestRewriteDuplicate(t *testing.T) {
+	buf := bytes.NewBufferString("[general]\nfoo=bar==\nfoo=bar2==\n")
+	cfg := ini.Parse(buf)
+
+	if v := cfg.Get("general", "foo"); v != "bar2==" {
+		t.Errorf("incorrect get %q", v)
+	}
+
+	var out bytes.Buffer
+	cfg.Write(&out)
+
+	correct := `[general]
+foo=bar2==
+
+`
+
+	if s := out.String(); s != correct {
+		t.Errorf("Incorrect INI after set:\n%s", s)
+	}
+}

Godeps/_workspace/src/github.com/codegangsta/inject/.gitignore

@@ -0,0 +1,2 @@
+inject
+inject.test

Godeps/_workspace/src/github.com/codegangsta/inject/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Jeremy Saenz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Godeps/_workspace/src/github.com/codegangsta/inject/README.md

@@ -0,0 +1,4 @@
+inject
+======
+
+Dependency injection for go

Godeps/_workspace/src/github.com/codegangsta/inject/inject.go

@@ -0,0 +1,168 @@
+// Package inject provides utilities for mapping and injecting dependencies in various ways.
+package inject
+
+import (
+	"fmt"
+	"reflect"
+)
+
+// Injector represents an interface for mapping and injecting dependencies into structs
+// and function arguments.
+type Injector interface {
+	Applicator
+	Invoker
+	TypeMapper
+	// SetParent sets the parent of the injector. If the injector cannot find a
+	// dependency in its Type map it will check its parent before returning an
+	// error.
+	SetParent(Injector)
+}
+
+// Applicator represents an interface for mapping dependencies to a struct.
+type Applicator interface {
+	// Maps dependencies in the Type map to each field in the struct
+	// that is tagged with 'inject'. Returns an error if the injection
+	// fails.
+	Apply(interface{}) error
+}
+
+// Invoker represents an interface for calling functions via reflection.
+type Invoker interface {
+	// Invoke attempts to call the interface{} provided as a function,
+	// providing dependencies for function arguments based on Type. Returns
+	// a slice of reflect.Value representing the returned values of the function.
+	// Returns an error if the injection fails.
+	Invoke(interface{}) ([]reflect.Value, error)
+}
+
+// TypeMapper represents an interface for mapping interface{} values based on type.
+type TypeMapper interface {
+	// Maps the interface{} value based on its immediate type from reflect.TypeOf.
+	Map(interface{}) TypeMapper
+	// Maps the interface{} value based on the pointer of an Interface provided.
+	// This is really only useful for mapping a value as an interface, as interfaces
+	// cannot at this time be referenced directly without a pointer.
+	MapTo(interface{}, interface{}) TypeMapper
+	// Provides a possibility to directly insert a mapping based on type and value.
+	// This makes it possible to directly map type arguments not possible to instantiate
+	// with reflect like unidirectional channels.
+	Set(reflect.Type, reflect.Value) TypeMapper
+	// Returns the Value that is mapped to the current type. Returns a zeroed Value if
+	// the Type has not been mapped.
+	Get(reflect.Type) reflect.Value
+}
+
+type injector struct {
+	values map[reflect.Type]reflect.Value
+	parent Injector
+}
+
+// InterfaceOf dereferences a pointer to an Interface type.
+// It panics if value is not an pointer to an interface.
+func InterfaceOf(value interface{}) reflect.Type {
+	t := reflect.TypeOf(value)
+
+	for t.Kind() == reflect.Ptr {
+		t = t.Elem()
+	}
+
+	if t.Kind() != reflect.Interface {
+		panic("Called inject.InterfaceOf with a value that is not a pointer to an interface. (*MyInterface)(nil)")
+	}
+
+	return t
+}
+
+// New returns a new Injector.
+func New() Injector {
+	return &injector{
+		values: make(map[reflect.Type]reflect.Value),
+	}
+}
+
+// Invoke attempts to call the interface{} provided as a function,
+// providing dependencies for function arguments based on Type.
+// Returns a slice of reflect.Value representing the returned values of the function.
+// Returns an error if the injection fails.
+// It panics if f is not a function
+func (inj *injector) Invoke(f interface{}) ([]reflect.Value, error) {
+	t := reflect.TypeOf(f)
+
+	var in = make([]reflect.Value, t.NumIn()) //Panic if t is not kind of Func
+	for i := 0; i < t.NumIn(); i++ {
+		argType := t.In(i)
+		val := inj.Get(argType)
+		if !val.IsValid() {
+			return nil, fmt.Errorf("Value not found for type %v", argType)
+		}
+
+		in[i] = val
+	}
+
+	return reflect.ValueOf(f).Call(in), nil
+}
+
+// Maps dependencies in the Type map to each field in the struct
+// that is tagged with 'inject'.
+// Returns an error if the injection fails.
+func (inj *injector) Apply(val interface{}) error {
+	v := reflect.ValueOf(val)
+
+	for v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+
+	if v.Kind() != reflect.Struct {
+		return nil // Should not panic here ?
+	}
+
+	t := v.Type()
+
+	for i := 0; i < v.NumField(); i++ {
+		f := v.Field(i)
+		structField := t.Field(i)
+		if f.CanSet() && structField.Tag == "inject" {
+			ft := f.Type()
+			v := inj.Get(ft)
+			if !v.IsValid() {
+				return fmt.Errorf("Value not found for type %v", ft)
+			}
+
+			f.Set(v)
+		}
+
+	}
+
+	return nil
+}
+
+// Maps the concrete value of val to its dynamic type using reflect.TypeOf,
+// It returns the TypeMapper registered in.
+func (i *injector) Map(val interface{}) TypeMapper {
+	i.values[reflect.TypeOf(val)] = reflect.ValueOf(val)
+	return i
+}
+
+func (i *injector) MapTo(val interface{}, ifacePtr interface{}) TypeMapper {
+	i.values[InterfaceOf(ifacePtr)] = reflect.ValueOf(val)
+	return i
+}
+
+// Maps the given reflect.Type to the given reflect.Value and returns
+// the Typemapper the mapping has been registered in.
+func (i *injector) Set(typ reflect.Type, val reflect.Value) TypeMapper {
+	i.values[typ] = val
+	return i
+}
+
+func (i *injector) Get(t reflect.Type) reflect.Value {
+	val := i.values[t]
+	if !val.IsValid() && i.parent != nil {
+		val = i.parent.Get(t)
+	}
+	return val
+}
+
+func (i *injector) SetParent(parent Injector) {
+	i.parent = parent
+}

Godeps/_workspace/src/github.com/codegangsta/inject/inject_test.go

@@ -0,0 +1,142 @@
+package inject_test
+
+import (
+	"github.com/codegangsta/inject"
+	"reflect"
+	"testing"
+)
+
+type SpecialString interface {
+}
+
+type TestStruct struct {
+	Dep1 string        `inject`
+	Dep2 SpecialString `inject`
+	Dep3 string
+}
+
+/* Test Helpers */
+func expect(t *testing.T, a interface{}, b interface{}) {
+	if a != b {
+		t.Errorf("Expected %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a))
+	}
+}
+
+func refute(t *testing.T, a interface{}, b interface{}) {
+	if a == b {
+		t.Errorf("Did not expect %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a))
+	}
+}
+
+func Test_InjectorInvoke(t *testing.T) {
+	injector := inject.New()
+	expect(t, injector == nil, false)
+
+	dep := "some dependency"
+	injector.Map(dep)
+	dep2 := "another dep"
+	injector.MapTo(dep2, (*SpecialString)(nil))
+	dep3 := make(chan *SpecialString)
+	dep4 := make(chan *SpecialString)
+	typRecv := reflect.ChanOf(reflect.RecvDir, reflect.TypeOf(dep3).Elem())
+	typSend := reflect.ChanOf(reflect.SendDir, reflect.TypeOf(dep4).Elem())
+	injector.Set(typRecv, reflect.ValueOf(dep3))
+	injector.Set(typSend, reflect.ValueOf(dep4))
+	
+	_, err := injector.Invoke(func(d1 string, d2 SpecialString, d3 <-chan *SpecialString, d4 chan<- *SpecialString) {
+		expect(t, d1, dep)
+		expect(t, d2, dep2)
+		expect(t, reflect.TypeOf(d3).Elem(), reflect.TypeOf(dep3).Elem())
+		expect(t, reflect.TypeOf(d4).Elem(), reflect.TypeOf(dep4).Elem())
+		expect(t, reflect.TypeOf(d3).ChanDir(), reflect.RecvDir)
+		expect(t, reflect.TypeOf(d4).ChanDir(), reflect.SendDir)
+	})
+
+	expect(t, err, nil)
+}
+
+func Test_InjectorInvokeReturnValues(t *testing.T) {
+	injector := inject.New()
+	expect(t, injector == nil, false)
+
+	dep := "some dependency"
+	injector.Map(dep)
+	dep2 := "another dep"
+	injector.MapTo(dep2, (*SpecialString)(nil))
+
+	result, err := injector.Invoke(func(d1 string, d2 SpecialString) string {
+		expect(t, d1, dep)
+		expect(t, d2, dep2)
+		return "Hello world"
+	})
+
+	expect(t, result[0].String(), "Hello world")
+	expect(t, err, nil)
+}
+
+func Test_InjectorApply(t *testing.T) {
+	injector := inject.New()
+
+	injector.Map("a dep").MapTo("another dep", (*SpecialString)(nil))
+
+	s := TestStruct{}
+	err := injector.Apply(&s)
+	expect(t, err, nil)
+
+	expect(t, s.Dep1, "a dep")
+	expect(t, s.Dep2, "another dep")
+}
+
+func Test_InterfaceOf(t *testing.T) {
+	iType := inject.InterfaceOf((*SpecialString)(nil))
+	expect(t, iType.Kind(), reflect.Interface)
+
+	iType = inject.InterfaceOf((**SpecialString)(nil))
+	expect(t, iType.Kind(), reflect.Interface)
+
+	// Expecting nil
+	defer func() {
+		rec := recover()
+		refute(t, rec, nil)
+	}()
+	iType = inject.InterfaceOf((*testing.T)(nil))
+}
+
+func Test_InjectorSet(t *testing.T) {
+	injector := inject.New()
+	typ      := reflect.TypeOf("string")
+	typSend  := reflect.ChanOf(reflect.SendDir, typ)
+	typRecv  := reflect.ChanOf(reflect.RecvDir, typ)
+	
+	// instantiating unidirectional channels is not possible using reflect
+	// http://golang.org/src/pkg/reflect/value.go?s=60463:60504#L2064
+	chanRecv := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, typ), 0)
+	chanSend := reflect.MakeChan(reflect.ChanOf(reflect.BothDir, typ), 0)
+	
+	injector.Set(typSend, chanSend)
+	injector.Set(typRecv, chanRecv)
+
+	expect(t, injector.Get(typSend).IsValid(), true)
+	expect(t, injector.Get(typRecv).IsValid(), true)
+	expect(t, injector.Get(chanSend.Type()).IsValid(), false)
+}
+
+
+func Test_InjectorGet(t *testing.T) {
+	injector := inject.New()
+
+	injector.Map("some dependency")
+
+	expect(t, injector.Get(reflect.TypeOf("string")).IsValid(), true)
+	expect(t, injector.Get(reflect.TypeOf(11)).IsValid(), false)
+}
+
+func Test_InjectorSetParent(t *testing.T) {
+	injector := inject.New()
+	injector.MapTo("another dep", (*SpecialString)(nil))
+
+	injector2 := inject.New()
+	injector2.SetParent(injector)
+
+	expect(t, injector2.Get(inject.InterfaceOf((*SpecialString)(nil))).IsValid(), true)
+}

Godeps/_workspace/src/github.com/codegangsta/martini/.gitignore

@@ -0,0 +1,23 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test

Godeps/_workspace/src/github.com/codegangsta/martini/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Jeremy Saenz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Godeps/_workspace/src/github.com/codegangsta/martini/README.md

@@ -0,0 +1,345 @@
+# Martini  [![wercker status](https://app.wercker.com/status/174bef7e3c999e103cacfe2770102266 "wercker status")](https://app.wercker.com/project/bykey/174bef7e3c999e103cacfe2770102266) [![GoDoc](https://godoc.org/github.com/codegangsta/martini?status.png)](http://godoc.org/github.com/codegangsta/martini)
+
+Martini is a powerful package for quickly writing modular web applications/services in Golang.
+
+Language Translations: [Simplified Chinese (zh_CN)](translations/README_zh_cn.md)
+
+
+## Getting Started
+
+After installing Go and setting up your [GOPATH](http://golang.org/doc/code.html#GOPATH), create your first `.go` file. We'll call it `server.go`.
+
+~~~ go
+package main
+
+import "github.com/codegangsta/martini"
+
+func main() {
+  m := martini.Classic()
+  m.Get("/", func() string {
+    return "Hello world!"
+  })
+  m.Run()
+}
+~~~
+
+Then install the Martini package (**go 1.1** and greater is required):
+~~~
+go get github.com/codegangsta/martini
+~~~
+
+Then run your server:
+~~~
+go run server.go
+~~~
+
+You will now have a Martini webserver running on `localhost:3000`.
+
+## Getting Help
+
+Join the [Mailing list](https://groups.google.com/forum/#!forum/martini-go)
+
+Watch the [Demo Video](http://martini.codegangsta.io/#demo)
+
+## Features
+* Extremely simple to use.
+* Non-intrusive design.
+* Plays nice with other Golang packages.
+* Awesome path matching and routing.
+* Modular design - Easy to add functionality, easy to rip stuff out.
+* Lots of good handlers/middlewares to use.
+* Great 'out of the box' feature set.
+* **Fully compatible with the [http.HandlerFunc](http://godoc.org/net/http#HandlerFunc) interface.**
+
+## More Middleware
+For more middleware and functionality, check out the repositories in the  [martini-contrib](https://github.com/martini-contrib) organization.
+
+## Table of Contents
+* [Classic Martini](#classic-martini)
+  * [Handlers](#handlers)
+  * [Routing](#routing)
+  * [Services](#services)
+  * [Serving Static Files](#serving-static-files)
+* [Middleware Handlers](#middleware-handlers)
+  * [Next()](#next)
+* [Martini Env](#martini-env)
+* [FAQ](#faq)
+
+## Classic Martini
+To get up and running quickly, [martini.Classic()](http://godoc.org/github.com/codegangsta/martini#Classic) provides some reasonable defaults that work well for most web applications:
+~~~ go
+  m := martini.Classic()
+  // ... middleware and routing goes here
+  m.Run()
+~~~
+
+Below is some of the functionality [martini.Classic()](http://godoc.org/github.com/codegangsta/martini#Classic) pulls in automatically:
+  * Request/Response Logging - [martini.Logger](http://godoc.org/github.com/codegangsta/martini#Logger)
+  * Panic Recovery - [martini.Recovery](http://godoc.org/github.com/codegangsta/martini#Recovery)
+  * Static File serving - [martini.Static](http://godoc.org/github.com/codegangsta/martini#Static)
+  * Routing - [martini.Router](http://godoc.org/github.com/codegangsta/martini#Router)
+
+### Handlers
+Handlers are the heart and soul of Martini. A handler is basically any kind of callable function:
+~~~ go
+m.Get("/", func() {
+  println("hello world")
+})
+~~~
+
+#### Return Values
+If a handler returns something, Martini will write the result to the current [http.ResponseWriter](http://godoc.org/net/http#ResponseWriter) as a string:
+~~~ go
+m.Get("/", func() string {
+  return "hello world" // HTTP 200 : "hello world"
+})
+~~~
+
+You can also optionally return a status code:
+~~~ go
+m.Get("/", func() (int, string) {
+  return 418, "i'm a teapot" // HTTP 418 : "i'm a teapot"
+})
+~~~
+
+#### Service Injection
+Handlers are invoked via reflection. Martini makes use of *Dependency Injection* to resolve dependencies in a Handlers argument list. **This makes Martini completely  compatible with golang's `http.HandlerFunc` interface.** 
+
+If you add an argument to your Handler, Martini will search its list of services and attempt to resolve the dependency via type assertion:
+~~~ go
+m.Get("/", func(res http.ResponseWriter, req *http.Request) { // res and req are injected by Martini
+  res.WriteHeader(200) // HTTP 200
+})
+~~~
+
+The following services are included with [martini.Classic()](http://godoc.org/github.com/codegangsta/martini#Classic):
+  * [*log.Logger](http://godoc.org/log#Logger) - Global logger for Martini.
+  * [martini.Context](http://godoc.org/github.com/codegangsta/martini#Context) - http request context.
+  * [martini.Params](http://godoc.org/github.com/codegangsta/martini#Params) - `map[string]string` of named params found by route matching.
+  * [martini.Routes](http://godoc.org/github.com/codegangsta/martini#Routes) - Route helper service.
+  * [http.ResponseWriter](http://godoc.org/net/http/#ResponseWriter) - http Response writer interface.
+  * [*http.Request](http://godoc.org/net/http/#Request) - http Request.
+
+### Routing
+In Martini, a route is an HTTP method paired with a URL-matching pattern.
+Each route can take one or more handler methods:
+~~~ go
+m.Get("/", func() {
+  // show something
+})
+
+m.Patch("/", func() {
+  // update something
+})
+
+m.Post("/", func() {
+  // create something
+})
+
+m.Put("/", func() {
+  // replace something
+})
+
+m.Delete("/", func() {
+  // destroy something
+})
+
+m.Options("/", func() {
+  // http options
+})
+
+m.NotFound(func() {
+  // handle 404
+})
+~~~
+
+Routes are matched in the order they are defined. The first route that
+matches the request is invoked.
+
+Route patterns may include named parameters, accessible via the [martini.Params](http://godoc.org/github.com/codegangsta/martini#Params) service:
+~~~ go
+m.Get("/hello/:name", func(params martini.Params) string {
+  return "Hello " + params["name"]
+})
+~~~
+
+Routes can be matched with regular expressions and globs as well:
+~~~ go
+m.Get("/hello/**", func(params martini.Params) string {
+  return "Hello " + params["_1"]
+})
+~~~
+
+Route handlers can be stacked on top of each other, which is useful for things like authentication and authorization:
+~~~ go
+m.Get("/secret", authorize, func() {
+  // this will execute as long as authorize doesn't write a response
+})
+~~~
+
+Route groups can be added too using the Group method.
+~~~ go
+m.Group("/books", func(r Router) {
+    r.Get("/:id", GetBooks)
+    r.Post("/new", NewBook)
+    r.Put("/update/:id", UpdateBook)
+    r.Delete("/delete/:id", DeleteBook)
+})
+~~~
+
+Just like you can pass middlewares to a handler you can pass middlewares to groups.
+~~~ go
+m.Group("/books", func(r Router) {
+    r.Get("/:id", GetBooks)
+    r.Post("/new", NewBook)
+    r.Put("/update/:id", UpdateBook)
+    r.Delete("/delete/:id", DeleteBook)
+}, MyMiddleware1, MyMiddleware2)
+~~~
+
+### Services
+Services are objects that are available to be injected into a Handler's argument list. You can map a service on a *Global* or *Request* level.
+
+#### Global Mapping
+A Martini instance implements the inject.Injector interface, so mapping a service is easy:
+~~~ go
+db := &MyDatabase{}
+m := martini.Classic()
+m.Map(db) // the service will be available to all handlers as *MyDatabase
+// ...
+m.Run()
+~~~
+
+#### Request-Level Mapping
+Mapping on the request level can be done in a handler via [martini.Context](http://godoc.org/github.com/codegangsta/martini#Context):
+~~~ go
+func MyCustomLoggerHandler(c martini.Context, req *http.Request) {
+  logger := &MyCustomLogger{req}
+  c.Map(logger) // mapped as *MyCustomLogger
+}
+~~~
+
+#### Mapping values to Interfaces
+One of the most powerful parts about services is the ability to map a service to an interface. For instance, if you wanted to override the [http.ResponseWriter](http://godoc.org/net/http#ResponseWriter) with an object that wrapped it and performed extra operations, you can write the following handler:
+~~~ go
+func WrapResponseWriter(res http.ResponseWriter, c martini.Context) {
+  rw := NewSpecialResponseWriter(res)
+  c.MapTo(rw, (*http.ResponseWriter)(nil)) // override ResponseWriter with our wrapper ResponseWriter
+}
+~~~
+
+### Serving Static Files
+A [martini.Classic()](http://godoc.org/github.com/codegangsta/martini#Classic) instance automatically serves static files from the "public" directory in the root of your server.
+You can serve from more directories by adding more [martini.Static](http://godoc.org/github.com/codegangsta/martini#Static) handlers.
+~~~ go
+m.Use(martini.Static("assets")) // serve from the "assets" directory as well
+~~~
+
+## Middleware Handlers
+Middleware Handlers sit between the incoming http request and the router. In essence they are no different than any other Handler in Martini. You can add a middleware handler to the stack like so:
+~~~ go
+m.Use(func() {
+  // do some middleware stuff
+})
+~~~
+
+You can have full control over the middleware stack with the `Handlers` function. This will replace any handlers that have been previously set:
+~~~ go
+m.Handlers(
+  Middleware1,
+  Middleware2,
+  Middleware3,
+)
+~~~
+
+Middleware Handlers work really well for things like logging, authorization, authentication, sessions, gzipping, error pages and any other operations that must happen before or after an http request:
+~~~ go
+// validate an api key
+m.Use(func(res http.ResponseWriter, req *http.Request) {
+  if req.Header.Get("X-API-KEY") != "secret123" {
+    res.WriteHeader(http.StatusUnauthorized)
+  }
+})
+~~~
+
+### Next()
+[Context.Next()](http://godoc.org/github.com/codegangsta/martini#Context) is an optional function that Middleware Handlers can call to yield the until after the other Handlers have been executed. This works really well for any operations that must happen after an http request:
+~~~ go
+// log before and after a request
+m.Use(func(c martini.Context, log *log.Logger){
+  log.Println("before a request")
+
+  c.Next()
+  
+  log.Println("after a request")
+})
+~~~
+
+## Martini Env
+
+Some Martini handlers make use of the `martini.Env` global variable to provide special functionality for development environments vs production environments. It is reccomended that the `MARTINI_ENV=production` environment variable to be set when deploying a Martini server into a production environment.
+
+## FAQ
+
+### Where do I find middleware X?
+
+Start by looking in the [martini-contrib](https://github.com/martini-contrib) projects. If it is not there feel free to contact a martini-contrib team member about adding a new repo to the organization.
+
+* [auth](https://github.com/martini-contrib/auth) - Handlers for authentication.
+* [binding](https://github.com/martini-contrib/binding) - Handler for mapping/validating a raw request into a structure.
+* [gzip](https://github.com/martini-contrib/gzip) - Handler for adding gzip compress to requests
+* [render](https://github.com/martini-contrib/render) - Handler that provides a service for easily rendering JSON and HTML templates.
+* [acceptlang](https://github.com/martini-contrib/acceptlang) - Handler for parsing the `Accept-Language` HTTP header.
+* [sessions](https://github.com/martini-contrib/sessions) - Handler that provides a Session service.
+* [strip](https://github.com/martini-contrib/strip) - URL Prefix stripping.
+* [method](https://github.com/martini-contrib/method) - HTTP method overriding via Header or form fields.
+* [secure](https://github.com/martini-contrib/secure) - Implements a few quick security wins.
+* [encoder](https://github.com/martini-contrib/encoder) - Encoder service for rendering data in several formats and content negotiation.
+* [cors](https://github.com/martini-contrib/cors) - Handler that enables CORS support.
+* [oauth2](https://github.com/martini-contrib/oauth2) - Handler that provides OAuth 2.0 login for Martini apps. Google Sign-in, Facebook Connect and Github login is supported.
+
+### How do I integrate with existing servers?
+
+A Martini instance implements `http.Handler`, so it can easily be used to serve subtrees 
+on existing Go servers. For example this is a working Martini app for Google App Engine:
+
+~~~ go
+package hello
+
+import (
+  "net/http"
+  "github.com/codegangsta/martini"
+)
+
+func init() {
+  m := martini.Classic()
+  m.Get("/", func() string {
+    return "Hello world!"
+  })
+  http.Handle("/", m)
+}
+~~~
+
+### How do I change the port/host?
+
+Martini's `Run` function looks for the PORT and HOST environment variables and uses those. Otherwise Martini will default to localhost:3000.
+To have more flexibility over port and host, use the `http.ListenAndServe` function instead.
+
+~~~ go
+  m := martini.Classic()
+  // ...
+  log.Fatal(http.ListenAndServe(":8080", m))
+~~~
+
+### Live code reload?
+
+[gin](https://github.com/codegangsta/gin) and [fresh](https://github.com/pilu/fresh) both live reload martini apps.
+
+## Contributing
+Martini is meant to be kept tiny and clean. Most contributions should end up in a repository in the [martini-contrib](https://github.com/martini-contrib) organization. If you do have a contribution for the core of Martini feel free to put up a Pull Request.
+
+## About
+
+Inspired by [express](https://github.com/visionmedia/express) and [sinatra](https://github.com/sinatra/sinatra)
+
+Martini is obsessively designed by none other than the [Code Gangsta](http://codegangsta.io/)

Godeps/_workspace/src/github.com/codegangsta/martini/env.go

@@ -0,0 +1,25 @@
+package martini
+
+import (
+	"os"
+)
+
+// Envs
+const (
+	Dev  string = "development"
+	Prod string = "production"
+	Test string = "test"
+)
+
+// Env is the environment that Martini is executing in. The MARTINI_ENV is read on initialization to set this variable.
+var Env = Dev
+
+func setENV(e string) {
+	if len(e) > 0 {
+		Env = e
+	}
+}
+
+func init() {
+	setENV(os.Getenv("MARTINI_ENV"))
+}

Godeps/_workspace/src/github.com/codegangsta/martini/env_test.go

@@ -0,0 +1,22 @@
+package martini
+
+import (
+	"testing"
+)
+
+func Test_SetENV(t *testing.T) {
+	tests := []struct {
+		in  string
+		out string
+	}{
+		{"", "development"},
+		{"not_development", "not_development"},
+	}
+
+	for _, test := range tests {
+		setENV(test.in)
+		if Env != test.out {
+			expect(t, Env, test.out)
+		}
+	}
+}

Godeps/_workspace/src/github.com/codegangsta/martini/go_version.go

@@ -0,0 +1,7 @@
+// +build !go1.1
+
+package martini
+
+func MartiniDoesNotSupportGo1Point0() {
+	"Martini requires Go 1.1 or greater."
+}

Godeps/_workspace/src/github.com/codegangsta/martini/logger.go

@@ -0,0 +1,20 @@
+package martini
+
+import (
+	"log"
+	"net/http"
+	"time"
+)
+
+// Logger returns a middleware handler that logs the request as it goes in and the response as it goes out.
+func Logger() Handler {
+	return func(res http.ResponseWriter, req *http.Request, c Context, log *log.Logger) {
+		start := time.Now()
+		log.Printf("Started %s %s", req.Method, req.URL.Path)
+
+		rw := res.(ResponseWriter)
+		c.Next()
+
+		log.Printf("Completed %v %s in %v\n", rw.Status(), http.StatusText(rw.Status()), time.Since(start))
+	}
+}

Godeps/_workspace/src/github.com/codegangsta/martini/logger_test.go

@@ -0,0 +1,31 @@
+package martini
+
+import (
+	"bytes"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func Test_Logger(t *testing.T) {
+	buff := bytes.NewBufferString("")
+	recorder := httptest.NewRecorder()
+
+	m := New()
+	// replace log for testing
+	m.Map(log.New(buff, "[martini] ", 0))
+	m.Use(Logger())
+	m.Use(func(res http.ResponseWriter) {
+		res.WriteHeader(http.StatusNotFound)
+	})
+
+	req, err := http.NewRequest("GET", "http://localhost:3000/foobar", nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	m.ServeHTTP(recorder, req)
+	expect(t, recorder.Code, http.StatusNotFound)
+	refute(t, len(buff.String()), 0)
+}

Godeps/_workspace/src/github.com/codegangsta/martini/martini.go

@@ -0,0 +1,173 @@
+// Package martini is a powerful package for quickly writing modular web applications/services in Golang.
+//
+// For a full guide visit http://github.com/codegangsta/martini
+//
+//  package main
+//
+//  import "github.com/codegangsta/martini"
+//
+//  func main() {
+//    m := martini.Classic()
+//
+//    m.Get("/", func() string {
+//      return "Hello world!"
+//    })
+//
+//    m.Run()
+//  }
+package martini
+
+import (
+	"log"
+	"net/http"
+	"os"
+	"reflect"
+
+	"github.com/codegangsta/inject"
+)
+
+// Martini represents the top level web application. inject.Injector methods can be invoked to map services on a global level.
+type Martini struct {
+	inject.Injector
+	handlers []Handler
+	action   Handler
+	logger   *log.Logger
+}
+
+// New creates a bare bones Martini instance. Use this method if you want to have full control over the middleware that is used.
+func New() *Martini {
+	m := &Martini{Injector: inject.New(), action: func() {}, logger: log.New(os.Stdout, "[martini] ", 0)}
+	m.Map(m.logger)
+	m.Map(defaultReturnHandler())
+	return m
+}
+
+// Handlers sets the entire middleware stack with the given Handlers. This will clear any current middleware handlers.
+// Will panic if any of the handlers is not a callable function
+func (m *Martini) Handlers(handlers ...Handler) {
+	m.handlers = make([]Handler, 0)
+	for _, handler := range handlers {
+		m.Use(handler)
+	}
+}
+
+// Action sets the handler that will be called after all the middleware has been invoked. This is set to martini.Router in a martini.Classic().
+func (m *Martini) Action(handler Handler) {
+	validateHandler(handler)
+	m.action = handler
+}
+
+// Use adds a middleware Handler to the stack. Will panic if the handler is not a callable func. Middleware Handlers are invoked in the order that they are added.
+func (m *Martini) Use(handler Handler) {
+	validateHandler(handler)
+
+	m.handlers = append(m.handlers, handler)
+}
+
+// ServeHTTP is the HTTP Entry point for a Martini instance. Useful if you want to control your own HTTP server.
+func (m *Martini) ServeHTTP(res http.ResponseWriter, req *http.Request) {
+	m.createContext(res, req).run()
+}
+
+// Run the http server. Listening on os.GetEnv("PORT") or 3000 by default.
+func (m *Martini) Run() {
+	port := os.Getenv("PORT")
+	if port == "" {
+		port = "3000"
+	}
+
+	host := os.Getenv("HOST")
+
+	m.logger.Println("listening on " + host + ":" + port)
+	m.logger.Fatalln(http.ListenAndServe(host+":"+port, m))
+}
+
+func (m *Martini) createContext(res http.ResponseWriter, req *http.Request) *context {
+	c := &context{inject.New(), m.handlers, m.action, NewResponseWriter(res), 0}
+	c.SetParent(m)
+	c.MapTo(c, (*Context)(nil))
+	c.MapTo(c.rw, (*http.ResponseWriter)(nil))
+	c.Map(req)
+	return c
+}
+
+// ClassicMartini represents a Martini with some reasonable defaults. Embeds the router functions for convenience.
+type ClassicMartini struct {
+	*Martini
+	Router
+}
+
+// Classic creates a classic Martini with some basic default middleware - martini.Logger, martini.Recovery and martini.Static.
+// Classic also maps martini.Routes as a service.
+func Classic() *ClassicMartini {
+	r := NewRouter()
+	m := New()
+	m.Use(Logger())
+	m.Use(Recovery())
+	m.Use(Static("public"))
+	m.MapTo(r, (*Routes)(nil))
+	m.Action(r.Handle)
+	return &ClassicMartini{m, r}
+}
+
+// Handler can be any callable function. Martini attempts to inject services into the handler's argument list.
+// Martini will panic if an argument could not be fullfilled via dependency injection.
+type Handler interface{}
+
+func validateHandler(handler Handler) {
+	if reflect.TypeOf(handler).Kind() != reflect.Func {
+		panic("martini handler must be a callable func")
+	}
+}
+
+// Context represents a request context. Services can be mapped on the request level from this interface.
+type Context interface {
+	inject.Injector
+	// Next is an optional function that Middleware Handlers can call to yield the until after
+	// the other Handlers have been executed. This works really well for any operations that must
+	// happen after an http request
+	Next()
+	// Written returns whether or not the response for this context has been written.
+	Written() bool
+}
+
+type context struct {
+	inject.Injector
+	handlers []Handler
+	action   Handler
+	rw       ResponseWriter
+	index    int
+}
+
+func (c *context) handler() Handler {
+	if c.index < len(c.handlers) {
+		return c.handlers[c.index]
+	}
+	if c.index == len(c.handlers) {
+		return c.action
+	}
+	panic("invalid index for context handler")
+}
+
+func (c *context) Next() {
+	c.index += 1
+	c.run()
+}
+
+func (c *context) Written() bool {
+	return c.rw.Written()
+}
+
+func (c *context) run() {
+	for c.index <= len(c.handlers) {
+		_, err := c.Invoke(c.handler())
+		if err != nil {
+			panic(err)
+		}
+		c.index += 1
+
+		if c.Written() {
+			return
+		}
+	}
+}

Godeps/_workspace/src/github.com/codegangsta/martini/martini_test.go

@@ -0,0 +1,141 @@
+package martini
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"testing"
+)
+
+/* Test Helpers */
+func expect(t *testing.T, a interface{}, b interface{}) {
+	if a != b {
+		t.Errorf("Expected %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a))
+	}
+}
+
+func refute(t *testing.T, a interface{}, b interface{}) {
+	if a == b {
+		t.Errorf("Did not expect %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a))
+	}
+}
+
+func Test_New(t *testing.T) {
+	m := New()
+	if m == nil {
+		t.Error("martini.New() cannot return nil")
+	}
+}
+
+func Test_Martini_Run(t *testing.T) {
+	// just test that Run doesn't bomb
+	go New().Run()
+}
+
+func Test_Martini_ServeHTTP(t *testing.T) {
+	result := ""
+	response := httptest.NewRecorder()
+
+	m := New()
+	m.Use(func(c Context) {
+		result += "foo"
+		c.Next()
+		result += "ban"
+	})
+	m.Use(func(c Context) {
+		result += "bar"
+		c.Next()
+		result += "baz"
+	})
+	m.Action(func(res http.ResponseWriter, req *http.Request) {
+		result += "bat"
+		res.WriteHeader(http.StatusBadRequest)
+	})
+
+	m.ServeHTTP(response, (*http.Request)(nil))
+
+	expect(t, result, "foobarbatbazban")
+	expect(t, response.Code, http.StatusBadRequest)
+}
+
+func Test_Martini_Handlers(t *testing.T) {
+	result := ""
+	response := httptest.NewRecorder()
+
+	batman := func(c Context) {
+		result += "batman!"
+	}
+
+	m := New()
+	m.Use(func(c Context) {
+		result += "foo"
+		c.Next()
+		result += "ban"
+	})
+	m.Handlers(
+		batman,
+		batman,
+		batman,
+	)
+	m.Action(func(res http.ResponseWriter, req *http.Request) {
+		result += "bat"
+		res.WriteHeader(http.StatusBadRequest)
+	})
+
+	m.ServeHTTP(response, (*http.Request)(nil))
+
+	expect(t, result, "batman!batman!batman!bat")
+	expect(t, response.Code, http.StatusBadRequest)
+}
+
+func Test_Martini_EarlyWrite(t *testing.T) {
+	result := ""
+	response := httptest.NewRecorder()
+
+	m := New()
+	m.Use(func(res http.ResponseWriter) {
+		result += "foobar"
+		res.Write([]byte("Hello world"))
+	})
+	m.Use(func() {
+		result += "bat"
+	})
+	m.Action(func(res http.ResponseWriter) {
+		result += "baz"
+		res.WriteHeader(http.StatusBadRequest)
+	})
+
+	m.ServeHTTP(response, (*http.Request)(nil))
+
+	expect(t, result, "foobar")
+	expect(t, response.Code, http.StatusOK)
+}
+
+func Test_Martini_Written(t *testing.T) {
+	response := httptest.NewRecorder()
+
+	m := New()
+	m.Handlers(func(res http.ResponseWriter) {
+		res.WriteHeader(http.StatusOK)
+	})
+
+	ctx := m.createContext(response, (*http.Request)(nil))
+	expect(t, ctx.Written(), false)
+
+	ctx.run()
+	expect(t, ctx.Written(), true)
+}
+
+func Test_Martini_Basic_NoRace(t *testing.T) {
+	m := New()
+	handlers := []Handler{func() {}, func() {}}
+	// Ensure append will not realloc to trigger the race condition
+	m.handlers = handlers[:1]
+	req, _ := http.NewRequest("GET", "/", nil)
+	for i := 0; i < 2; i++ {
+		go func() {
+			response := httptest.NewRecorder()
+			m.ServeHTTP(response, req)
+		}()
+	}
+}

Godeps/_workspace/src/github.com/codegangsta/martini/recovery.go

@@ -0,0 +1,142 @@
+package martini
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"runtime"
+
+	"github.com/codegangsta/inject"
+)
+
+const (
+	panicHtml = `<html>
+<head><title>PANIC: %s</title>
+<style type="text/css">
+html, body {
+	font-family: "Roboto", sans-serif;
+	color: #333333;
+	background-color: #ea5343;
+	margin: 0px;
+}
+h1 {
+	color: #d04526;
+	background-color: #ffffff;
+	padding: 20px;
+	border-bottom: 1px dashed #2b3848;
+}
+pre {
+	margin: 20px;
+	padding: 20px;
+	border: 2px solid #2b3848;
+	background-color: #ffffff;
+}
+</style>
+</head><body>
+<h1>PANIC</h1>
+<pre style="font-weight: bold;">%s</pre>
+<pre>%s</pre>
+</body>
+</html>`
+)
+
+var (
+	dunno     = []byte("???")
+	centerDot = []byte("·")
+	dot       = []byte(".")
+	slash     = []byte("/")
+)
+
+// stack returns a nicely formated stack frame, skipping skip frames
+func stack(skip int) []byte {
+	buf := new(bytes.Buffer) // the returned data
+	// As we loop, we open files and read them. These variables record the currently
+	// loaded file.
+	var lines [][]byte
+	var lastFile string
+	for i := skip; ; i++ { // Skip the expected number of frames
+		pc, file, line, ok := runtime.Caller(i)
+		if !ok {
+			break
+		}
+		// Print this much at least.  If we can't find the source, it won't show.
+		fmt.Fprintf(buf, "%s:%d (0x%x)\n", file, line, pc)
+		if file != lastFile {
+			data, err := ioutil.ReadFile(file)
+			if err != nil {
+				continue
+			}
+			lines = bytes.Split(data, []byte{'\n'})
+			lastFile = file
+		}
+		fmt.Fprintf(buf, "\t%s: %s\n", function(pc), source(lines, line))
+	}
+	return buf.Bytes()
+}
+
+// source returns a space-trimmed slice of the n'th line.
+func source(lines [][]byte, n int) []byte {
+	n-- // in stack trace, lines are 1-indexed but our array is 0-indexed
+	if n < 0 || n >= len(lines) {
+		return dunno
+	}
+	return bytes.TrimSpace(lines[n])
+}
+
+// function returns, if possible, the name of the function containing the PC.
+func function(pc uintptr) []byte {
+	fn := runtime.FuncForPC(pc)
+	if fn == nil {
+		return dunno
+	}
+	name := []byte(fn.Name())
+	// The name includes the path name to the package, which is unnecessary
+	// since the file name is already included.  Plus, it has center dots.
+	// That is, we see
+	//	runtime/debug.*T·ptrmethod
+	// and want
+	//	*T.ptrmethod
+	// Also the package path might contains dot (e.g. code.google.com/...),
+	// so first eliminate the path prefix
+	if lastslash := bytes.LastIndex(name, slash); lastslash >= 0 {
+		name = name[lastslash+1:]
+	}
+	if period := bytes.Index(name, dot); period >= 0 {
+		name = name[period+1:]
+	}
+	name = bytes.Replace(name, centerDot, dot, -1)
+	return name
+}
+
+// Recovery returns a middleware that recovers from any panics and writes a 500 if there was one.
+// While Martini is in development mode, Recovery will also output the panic as HTML.
+func Recovery() Handler {
+	return func(c Context, log *log.Logger) {
+		defer func() {
+			if err := recover(); err != nil {
+				stack := stack(3)
+				log.Printf("PANIC: %s\n%s", err, stack)
+
+				// Lookup the current responsewriter
+				val := c.Get(inject.InterfaceOf((*http.ResponseWriter)(nil)))
+				res := val.Interface().(http.ResponseWriter)
+
+				// respond with panic message while in development mode
+				var body []byte
+				if Env == Dev {
+					res.Header().Set("Content-Type", "text/html")
+					body = []byte(fmt.Sprintf(panicHtml, err, err, stack))
+				}
+
+				res.WriteHeader(http.StatusInternalServerError)
+				if nil != body {
+					res.Write(body)
+				}
+			}
+		}()
+
+		c.Next()
+	}
+}

Godeps/_workspace/src/github.com/codegangsta/martini/recovery_test.go

@@ -0,0 +1,49 @@
+package martini
+
+import (
+	"bytes"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func Test_Recovery(t *testing.T) {
+	buff := bytes.NewBufferString("")
+	recorder := httptest.NewRecorder()
+
+	setENV(Dev)
+	m := New()
+	// replace log for testing
+	m.Map(log.New(buff, "[martini] ", 0))
+	m.Use(func(res http.ResponseWriter, req *http.Request) {
+		res.Header().Set("Content-Type", "unpredictable")
+	})
+	m.Use(Recovery())
+	m.Use(func(res http.ResponseWriter, req *http.Request) {
+		panic("here is a panic!")
+	})
+	m.ServeHTTP(recorder, (*http.Request)(nil))
+	expect(t, recorder.Code, http.StatusInternalServerError)
+	expect(t, recorder.HeaderMap.Get("Content-Type"), "text/html")
+	refute(t, recorder.Body.Len(), 0)
+	refute(t, len(buff.String()), 0)
+}
+
+func Test_Recovery_ResponseWriter(t *testing.T) {
+	recorder := httptest.NewRecorder()
+	recorder2 := httptest.NewRecorder()
+
+	setENV(Dev)
+	m := New()
+	m.Use(Recovery())
+	m.Use(func(c Context) {
+		c.MapTo(recorder2, (*http.ResponseWriter)(nil))
+		panic("here is a panic!")
+	})
+	m.ServeHTTP(recorder, (*http.Request)(nil))
+
+	expect(t, recorder2.Code, http.StatusInternalServerError)
+	expect(t, recorder2.HeaderMap.Get("Content-Type"), "text/html")
+	refute(t, recorder2.Body.Len(), 0)
+}

Godeps/_workspace/src/github.com/codegangsta/martini/response_writer.go

@@ -0,0 +1,97 @@
+package martini
+
+import (
+	"bufio"
+	"fmt"
+	"net"
+	"net/http"
+)
+
+// ResponseWriter is a wrapper around http.ResponseWriter that provides extra information about
+// the response. It is recommended that middleware handlers use this construct to wrap a responsewriter
+// if the functionality calls for it.
+type ResponseWriter interface {
+	http.ResponseWriter
+	http.Flusher
+	// Status returns the status code of the response or 0 if the response has not been written.
+	Status() int
+	// Written returns whether or not the ResponseWriter has been written.
+	Written() bool
+	// Size returns the size of the response body.
+	Size() int
+	// Before allows for a function to be called before the ResponseWriter has been written to. This is
+	// useful for setting headers or any other operations that must happen before a response has been written.
+	Before(BeforeFunc)
+}
+
+// BeforeFunc is a function that is called before the ResponseWriter has been written to.
+type BeforeFunc func(ResponseWriter)
+
+// NewResponseWriter creates a ResponseWriter that wraps an http.ResponseWriter
+func NewResponseWriter(rw http.ResponseWriter) ResponseWriter {
+	return &responseWriter{rw, 0, 0, nil}
+}
+
+type responseWriter struct {
+	http.ResponseWriter
+	status      int
+	size        int
+	beforeFuncs []BeforeFunc
+}
+
+func (rw *responseWriter) WriteHeader(s int) {
+	rw.callBefore()
+	rw.ResponseWriter.WriteHeader(s)
+	rw.status = s
+}
+
+func (rw *responseWriter) Write(b []byte) (int, error) {
+	if !rw.Written() {
+		// The status will be StatusOK if WriteHeader has not been called yet
+		rw.WriteHeader(http.StatusOK)
+	}
+	size, err := rw.ResponseWriter.Write(b)
+	rw.size += size
+	return size, err
+}
+
+func (rw *responseWriter) Status() int {
+	return rw.status
+}
+
+func (rw *responseWriter) Size() int {
+	return rw.size
+}
+
+func (rw *responseWriter) Written() bool {
+	return rw.status != 0
+}
+
+func (rw *responseWriter) Before(before BeforeFunc) {
+	rw.beforeFuncs = append(rw.beforeFuncs, before)
+}
+
+func (rw *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	hijacker, ok := rw.ResponseWriter.(http.Hijacker)
+	if !ok {
+		return nil, nil, fmt.Errorf("the ResponseWriter doesn't support the Hijacker interface")
+	}
+	return hijacker.Hijack()
+}
+
+func (rw *responseWriter) CloseNotify() <-chan bool {
+	return rw.ResponseWriter.(http.CloseNotifier).CloseNotify()
+}
+
+func (rw *responseWriter) callBefore() {
+	for i := len(rw.beforeFuncs) - 1; i >= 0; i-- {
+		rw.beforeFuncs[i](rw)
+	}
+}
+
+func (rw *responseWriter) Flush() {
+	flusher, ok := rw.ResponseWriter.(http.Flusher)
+	if ok {
+		flusher.Flush()
+	}
+}

Godeps/_workspace/src/github.com/codegangsta/martini/response_writer_test.go

@@ -0,0 +1,188 @@
+package martini
+
+import (
+	"bufio"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+type closeNotifyingRecorder struct {
+	*httptest.ResponseRecorder
+	closed chan bool
+}
+
+func newCloseNotifyingRecorder() *closeNotifyingRecorder {
+	return &closeNotifyingRecorder{
+		httptest.NewRecorder(),
+		make(chan bool, 1),
+	}
+}
+
+func (c *closeNotifyingRecorder) close() {
+	c.closed <- true
+}
+
+func (c *closeNotifyingRecorder) CloseNotify() <-chan bool {
+	return c.closed
+}
+
+type hijackableResponse struct {
+	Hijacked bool
+}
+
+func newHijackableResponse() *hijackableResponse {
+	return &hijackableResponse{}
+}
+
+func (h *hijackableResponse) Header() http.Header           { return nil }
+func (h *hijackableResponse) Write(buf []byte) (int, error) { return 0, nil }
+func (h *hijackableResponse) WriteHeader(code int)          {}
+func (h *hijackableResponse) Flush()                        {}
+func (h *hijackableResponse) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	h.Hijacked = true
+	return nil, nil, nil
+}
+
+func Test_ResponseWriter_WritingString(t *testing.T) {
+	rec := httptest.NewRecorder()
+	rw := NewResponseWriter(rec)
+
+	rw.Write([]byte("Hello world"))
+
+	expect(t, rec.Code, rw.Status())
+	expect(t, rec.Body.String(), "Hello world")
+	expect(t, rw.Status(), http.StatusOK)
+	expect(t, rw.Size(), 11)
+	expect(t, rw.Written(), true)
+}
+
+func Test_ResponseWriter_WritingStrings(t *testing.T) {
+	rec := httptest.NewRecorder()
+	rw := NewResponseWriter(rec)
+
+	rw.Write([]byte("Hello world"))
+	rw.Write([]byte("foo bar bat baz"))
+
+	expect(t, rec.Code, rw.Status())
+	expect(t, rec.Body.String(), "Hello worldfoo bar bat baz")
+	expect(t, rw.Status(), http.StatusOK)
+	expect(t, rw.Size(), 26)
+}
+
+func Test_ResponseWriter_WritingHeader(t *testing.T) {
+	rec := httptest.NewRecorder()
+	rw := NewResponseWriter(rec)
+
+	rw.WriteHeader(http.StatusNotFound)
+
+	expect(t, rec.Code, rw.Status())
+	expect(t, rec.Body.String(), "")
+	expect(t, rw.Status(), http.StatusNotFound)
+	expect(t, rw.Size(), 0)
+}
+
+func Test_ResponseWriter_Before(t *testing.T) {
+	rec := httptest.NewRecorder()
+	rw := NewResponseWriter(rec)
+	result := ""
+
+	rw.Before(func(ResponseWriter) {
+		result += "foo"
+	})
+	rw.Before(func(ResponseWriter) {
+		result += "bar"
+	})
+
+	rw.WriteHeader(http.StatusNotFound)
+
+	expect(t, rec.Code, rw.Status())
+	expect(t, rec.Body.String(), "")
+	expect(t, rw.Status(), http.StatusNotFound)
+	expect(t, rw.Size(), 0)
+	expect(t, result, "barfoo")
+}
+
+func Test_ResponseWriter_Hijack(t *testing.T) {
+	hijackable := newHijackableResponse()
+	rw := NewResponseWriter(hijackable)
+	hijacker, ok := rw.(http.Hijacker)
+	expect(t, ok, true)
+	_, _, err := hijacker.Hijack()
+	if err != nil {
+		t.Error(err)
+	}
+	expect(t, hijackable.Hijacked, true)
+}
+
+func Test_ResponseWrite_Hijack_NotOK(t *testing.T) {
+	hijackable := new(http.ResponseWriter)
+	rw := NewResponseWriter(*hijackable)
+	hijacker, ok := rw.(http.Hijacker)
+	expect(t, ok, true)
+	_, _, err := hijacker.Hijack()
+
+	refute(t, err, nil)
+}
+
+func Test_ResponseWriter_CloseNotify(t *testing.T) {
+	rec := newCloseNotifyingRecorder()
+	rw := NewResponseWriter(rec)
+	closed := false
+	notifier := rw.(http.CloseNotifier).CloseNotify()
+	rec.close()
+	select {
+	case <-notifier:
+		closed = true
+	case <-time.After(time.Second):
+	}
+	expect(t, closed, true)
+}
+
+func Test_ResponseWriter_Flusher(t *testing.T) {
+
+	rec := httptest.NewRecorder()
+	rw := NewResponseWriter(rec)
+
+	_, ok := rw.(http.Flusher)
+	expect(t, ok, true)
+}
+
+func Test_ResponseWriter_FlusherHandler(t *testing.T) {
+
+	// New martini instance
+	m := Classic()
+
+	m.Get("/events", func(w http.ResponseWriter, r *http.Request) {
+
+		f, ok := w.(http.Flusher)
+		expect(t, ok, true)
+
+		w.Header().Set("Content-Type", "text/event-stream")
+		w.Header().Set("Cache-Control", "no-cache")
+		w.Header().Set("Connection", "keep-alive")
+
+		for i := 0; i < 2; i++ {
+			time.Sleep(10 * time.Millisecond)
+			io.WriteString(w, "data: Hello\n\n")
+			f.Flush()
+		}
+
+	})
+
+	recorder := httptest.NewRecorder()
+	r, _ := http.NewRequest("GET", "/events", nil)
+	m.ServeHTTP(recorder, r)
+
+	if recorder.Code != 200 {
+		t.Error("Response not 200")
+	}
+
+	if recorder.Body.String() != "data: Hello\n\ndata: Hello\n\n" {
+		t.Error("Didn't receive correct body, got:", recorder.Body.String())
+	}
+
+}

Godeps/_workspace/src/github.com/codegangsta/martini/return_handler.go

@@ -0,0 +1,43 @@
+package martini
+
+import (
+	"github.com/codegangsta/inject"
+	"net/http"
+	"reflect"
+)
+
+// ReturnHandler is a service that Martini provides that is called
+// when a route handler returns something. The ReturnHandler is
+// responsible for writing to the ResponseWriter based on the values
+// that are passed into this function.
+type ReturnHandler func(Context, []reflect.Value)
+
+func defaultReturnHandler() ReturnHandler {
+	return func(ctx Context, vals []reflect.Value) {
+		rv := ctx.Get(inject.InterfaceOf((*http.ResponseWriter)(nil)))
+		res := rv.Interface().(http.ResponseWriter)
+		var responseVal reflect.Value
+		if len(vals) > 1 && vals[0].Kind() == reflect.Int {
+			res.WriteHeader(int(vals[0].Int()))
+			responseVal = vals[1]
+		} else if len(vals) > 0 {
+			responseVal = vals[0]
+		}
+		if canDeref(responseVal) {
+			responseVal = responseVal.Elem()
+		}
+		if isByteSlice(responseVal) {
+			res.Write(responseVal.Bytes())
+		} else {
+			res.Write([]byte(responseVal.String()))
+		}
+	}
+}
+
+func isByteSlice(val reflect.Value) bool {
+	return val.Kind() == reflect.Slice && val.Type().Elem().Kind() == reflect.Uint8
+}
+
+func canDeref(val reflect.Value) bool {
+	return val.Kind() == reflect.Interface || val.Kind() == reflect.Ptr
+}

Godeps/_workspace/src/github.com/codegangsta/martini/router.go

@@ -0,0 +1,331 @@
+package martini
+
+import (
+	"fmt"
+	"net/http"
+	"reflect"
+	"regexp"
+	"strconv"
+)
+
+// Params is a map of name/value pairs for named routes. An instance of martini.Params is available to be injected into any route handler.
+type Params map[string]string
+
+// Router is Martini's de-facto routing interface. Supports HTTP verbs, stacked handlers, and dependency injection.
+type Router interface {
+	Routes
+
+	// Group adds a group where related routes can be added.
+	Group(string, func(Router), ...Handler)
+	// Get adds a route for a HTTP GET request to the specified matching pattern.
+	Get(string, ...Handler) Route
+	// Patch adds a route for a HTTP PATCH request to the specified matching pattern.
+	Patch(string, ...Handler) Route
+	// Post adds a route for a HTTP POST request to the specified matching pattern.
+	Post(string, ...Handler) Route
+	// Put adds a route for a HTTP PUT request to the specified matching pattern.
+	Put(string, ...Handler) Route
+	// Delete adds a route for a HTTP DELETE request to the specified matching pattern.
+	Delete(string, ...Handler) Route
+	// Options adds a route for a HTTP OPTIONS request to the specified matching pattern.
+	Options(string, ...Handler) Route
+	// Head adds a route for a HTTP HEAD request to the specified matching pattern.
+	Head(string, ...Handler) Route
+	// Any adds a route for any HTTP method request to the specified matching pattern.
+	Any(string, ...Handler) Route
+
+	// NotFound sets the handlers that are called when a no route matches a request. Throws a basic 404 by default.
+	NotFound(...Handler)
+
+	// Handle is the entry point for routing. This is used as a martini.Handler
+	Handle(http.ResponseWriter, *http.Request, Context)
+}
+
+type router struct {
+	routes    []*route
+	notFounds []Handler
+	groups    []group
+}
+
+type group struct {
+	pattern  string
+	handlers []Handler
+}
+
+// NewRouter creates a new Router instance.
+// If you aren't using ClassicMartini, then you can add Routes as a
+// service with:
+//
+//	m := martini.New()
+//	r := martini.NewRouter()
+//	m.MapTo(r, (*martini.Routes)(nil))
+//
+// If you are using ClassicMartini, then this is done for you.
+func NewRouter() Router {
+	return &router{notFounds: []Handler{http.NotFound}, groups: make([]group, 0)}
+}
+
+func (r *router) Group(pattern string, fn func(Router), h ...Handler) {
+	r.groups = append(r.groups, group{pattern, h})
+	fn(r)
+	r.groups = r.groups[:len(r.groups)-1]
+}
+
+func (r *router) Get(pattern string, h ...Handler) Route {
+	return r.addRoute("GET", pattern, h)
+}
+
+func (r *router) Patch(pattern string, h ...Handler) Route {
+	return r.addRoute("PATCH", pattern, h)
+}
+
+func (r *router) Post(pattern string, h ...Handler) Route {
+	return r.addRoute("POST", pattern, h)
+}
+
+func (r *router) Put(pattern string, h ...Handler) Route {
+	return r.addRoute("PUT", pattern, h)
+}
+
+func (r *router) Delete(pattern string, h ...Handler) Route {
+	return r.addRoute("DELETE", pattern, h)
+}
+
+func (r *router) Options(pattern string, h ...Handler) Route {
+	return r.addRoute("OPTIONS", pattern, h)
+}
+
+func (r *router) Head(pattern string, h ...Handler) Route {
+	return r.addRoute("HEAD", pattern, h)
+}
+
+func (r *router) Any(pattern string, h ...Handler) Route {
+	return r.addRoute("*", pattern, h)
+}
+
+func (r *router) Handle(res http.ResponseWriter, req *http.Request, context Context) {
+	for _, route := range r.routes {
+		ok, vals := route.Match(req.Method, req.URL.Path)
+		if ok {
+			params := Params(vals)
+			context.Map(params)
+			route.Handle(context, res)
+			return
+		}
+	}
+
+	// no routes exist, 404
+	c := &routeContext{context, 0, r.notFounds}
+	context.MapTo(c, (*Context)(nil))
+	c.run()
+}
+
+func (r *router) NotFound(handler ...Handler) {
+	r.notFounds = handler
+}
+
+func (r *router) addRoute(method string, pattern string, handlers []Handler) *route {
+	if len(r.groups) > 0 {
+		group := r.groups[len(r.groups)-1]
+		pattern = group.pattern + pattern
+		h := make([]Handler, len(group.handlers)+len(handlers))
+		copy(h, group.handlers)
+		copy(h[len(group.handlers):], handlers)
+		handlers = h
+	}
+
+	route := newRoute(method, pattern, handlers)
+	route.Validate()
+	r.routes = append(r.routes, route)
+	return route
+}
+
+func (r *router) findRoute(name string) *route {
+	for _, route := range r.routes {
+		if route.name == name {
+			return route
+		}
+	}
+
+	return nil
+}
+
+// Route is an interface representing a Route in Martini's routing layer.
+type Route interface {
+	// URLWith returns a rendering of the Route's url with the given string params.
+	URLWith([]string) string
+	Name(string)
+}
+
+type route struct {
+	method   string
+	regex    *regexp.Regexp
+	handlers []Handler
+	pattern  string
+	name     string
+}
+
+func newRoute(method string, pattern string, handlers []Handler) *route {
+	route := route{method, nil, handlers, pattern, ""}
+	r := regexp.MustCompile(`:[^/#?()\.\\]+`)
+	pattern = r.ReplaceAllStringFunc(pattern, func(m string) string {
+		return fmt.Sprintf(`(?P<%s>[^/#?]+)`, m[1:])
+	})
+	r2 := regexp.MustCompile(`\*\*`)
+	var index int
+	pattern = r2.ReplaceAllStringFunc(pattern, func(m string) string {
+		index++
+		return fmt.Sprintf(`(?P<_%d>[^#?]*)`, index)
+	})
+	pattern += `\/?`
+	route.regex = regexp.MustCompile(pattern)
+	return &route
+}
+
+func (r route) MatchMethod(method string) bool {
+	return r.method == "*" || method == r.method || (method == "HEAD" && r.method == "GET")
+}
+
+func (r route) Match(method string, path string) (bool, map[string]string) {
+	// add Any method matching support
+	if !r.MatchMethod(method) {
+		return false, nil
+	}
+
+	matches := r.regex.FindStringSubmatch(path)
+	if len(matches) > 0 && matches[0] == path {
+		params := make(map[string]string)
+		for i, name := range r.regex.SubexpNames() {
+			if len(name) > 0 {
+				params[name] = matches[i]
+			}
+		}
+		return true, params
+	}
+	return false, nil
+}
+
+func (r *route) Validate() {
+	for _, handler := range r.handlers {
+		validateHandler(handler)
+	}
+}
+
+func (r *route) Handle(c Context, res http.ResponseWriter) {
+	context := &routeContext{c, 0, r.handlers}
+	c.MapTo(context, (*Context)(nil))
+	context.run()
+}
+
+// URLWith returns the url pattern replacing the parameters for its values
+func (r *route) URLWith(args []string) string {
+	if len(args) > 0 {
+		reg := regexp.MustCompile(`:[^/#?()\.\\]+`)
+		argCount := len(args)
+		i := 0
+		url := reg.ReplaceAllStringFunc(r.pattern, func(m string) string {
+			var val interface{}
+			if i < argCount {
+				val = args[i]
+			} else {
+				val = m
+			}
+			i += 1
+			return fmt.Sprintf(`%v`, val)
+		})
+
+		return url
+	}
+	return r.pattern
+}
+
+func (r *route) Name(name string) {
+	r.name = name
+}
+
+// Routes is a helper service for Martini's routing layer.
+type Routes interface {
+	// URLFor returns a rendered URL for the given route. Optional params can be passed to fulfill named parameters in the route.
+	URLFor(name string, params ...interface{}) string
+	// MethodsFor returns an array of methods available for the path
+	MethodsFor(path string) []string
+}
+
+// URLFor returns the url for the given route name.
+func (r *router) URLFor(name string, params ...interface{}) string {
+	route := r.findRoute(name)
+
+	if route == nil {
+		panic("route not found")
+	}
+
+	var args []string
+	for _, param := range params {
+		switch v := param.(type) {
+		case int:
+			args = append(args, strconv.FormatInt(int64(v), 10))
+		case string:
+			args = append(args, v)
+		default:
+			if v != nil {
+				panic("Arguments passed to URLFor must be integers or strings")
+			}
+		}
+	}
+
+	return route.URLWith(args)
+}
+
+func hasMethod(methods []string, method string) bool {
+	for _, v := range methods {
+		if v == method {
+			return true
+		}
+	}
+	return false
+}
+
+// MethodsFor returns all methods available for path
+func (r *router) MethodsFor(path string) []string {
+	methods := []string{}
+	for _, route := range r.routes {
+		matches := route.regex.FindStringSubmatch(path)
+		if len(matches) > 0 && matches[0] == path && !hasMethod(methods, route.method) {
+			methods = append(methods, route.method)
+		}
+	}
+	return methods
+}
+
+type routeContext struct {
+	Context
+	index    int
+	handlers []Handler
+}
+
+func (r *routeContext) Next() {
+	r.index += 1
+	r.run()
+}
+
+func (r *routeContext) run() {
+	for r.index < len(r.handlers) {
+		handler := r.handlers[r.index]
+		vals, err := r.Invoke(handler)
+		if err != nil {
+			panic(err)
+		}
+		r.index += 1
+
+		// if the handler returned something, write it to the http response
+		if len(vals) > 0 {
+			ev := r.Get(reflect.TypeOf(ReturnHandler(nil)))
+			handleReturn := ev.Interface().(ReturnHandler)
+			handleReturn(r, vals)
+		}
+
+		if r.Written() {
+			return
+		}
+	}
+}

Godeps/_workspace/src/github.com/codegangsta/martini/router_test.go

@@ -0,0 +1,410 @@
+package martini
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func Test_Routing(t *testing.T) {
+	router := NewRouter()
+	recorder := httptest.NewRecorder()
+
+	req, _ := http.NewRequest("GET", "http://localhost:3000/foo", nil)
+	context := New().createContext(recorder, req)
+
+	req2, _ := http.NewRequest("POST", "http://localhost:3000/bar/bat", nil)
+	context2 := New().createContext(recorder, req2)
+
+	req3, _ := http.NewRequest("DELETE", "http://localhost:3000/baz", nil)
+	context3 := New().createContext(recorder, req3)
+
+	req4, _ := http.NewRequest("PATCH", "http://localhost:3000/bar/foo", nil)
+	context4 := New().createContext(recorder, req4)
+
+	req5, _ := http.NewRequest("GET", "http://localhost:3000/fez/this/should/match", nil)
+	context5 := New().createContext(recorder, req5)
+
+	req6, _ := http.NewRequest("PUT", "http://localhost:3000/pop/blah/blah/blah/bap/foo/", nil)
+	context6 := New().createContext(recorder, req6)
+
+	req7, _ := http.NewRequest("DELETE", "http://localhost:3000/wap//pow", nil)
+	context7 := New().createContext(recorder, req7)
+
+	req8, _ := http.NewRequest("HEAD", "http://localhost:3000/wap//pow", nil)
+	context8 := New().createContext(recorder, req8)
+
+	req9, _ := http.NewRequest("OPTIONS", "http://localhost:3000/opts", nil)
+	context9 := New().createContext(recorder, req9)
+
+	req10, _ := http.NewRequest("HEAD", "http://localhost:3000/foo", nil)
+	context10 := New().createContext(recorder, req10)
+
+	req11, _ := http.NewRequest("GET", "http://localhost:3000/bazz/inga", nil)
+	context11 := New().createContext(recorder, req11)
+
+	req12, _ := http.NewRequest("POST", "http://localhost:3000/bazz/inga", nil)
+	context12 := New().createContext(recorder, req12)
+
+	result := ""
+	router.Get("/foo", func(req *http.Request) {
+		result += "foo"
+	})
+	router.Patch("/bar/:id", func(params Params) {
+		expect(t, params["id"], "foo")
+		result += "barfoo"
+	})
+	router.Post("/bar/:id", func(params Params) {
+		expect(t, params["id"], "bat")
+		result += "barbat"
+	})
+	router.Put("/fizzbuzz", func() {
+		result += "fizzbuzz"
+	})
+	router.Delete("/bazzer", func(c Context) {
+		result += "baz"
+	})
+	router.Get("/fez/**", func(params Params) {
+		expect(t, params["_1"], "this/should/match")
+		result += "fez"
+	})
+	router.Put("/pop/**/bap/:id/**", func(params Params) {
+		expect(t, params["id"], "foo")
+		expect(t, params["_1"], "blah/blah/blah")
+		expect(t, params["_2"], "")
+		result += "popbap"
+	})
+	router.Delete("/wap/**/pow", func(params Params) {
+		expect(t, params["_1"], "")
+		result += "wappow"
+	})
+	router.Options("/opts", func() {
+		result += "opts"
+	})
+	router.Head("/wap/**/pow", func(params Params) {
+		expect(t, params["_1"], "")
+		result += "wappow"
+	})
+	router.Group("/bazz", func(r Router) {
+		r.Get("/inga", func() {
+			result += "get"
+		})
+
+		r.Post("/inga", func() {
+			result += "post"
+		})
+	}, func() {
+		result += "bazz"
+	}, func() {
+		result += "inga"
+	})
+
+	router.Handle(recorder, req, context)
+	router.Handle(recorder, req2, context2)
+	router.Handle(recorder, req3, context3)
+	router.Handle(recorder, req4, context4)
+	router.Handle(recorder, req5, context5)
+	router.Handle(recorder, req6, context6)
+	router.Handle(recorder, req7, context7)
+	router.Handle(recorder, req8, context8)
+	router.Handle(recorder, req9, context9)
+	router.Handle(recorder, req10, context10)
+	router.Handle(recorder, req11, context11)
+	router.Handle(recorder, req12, context12)
+	expect(t, result, "foobarbatbarfoofezpopbapwappowwappowoptsfoobazzingagetbazzingapost")
+	expect(t, recorder.Code, http.StatusNotFound)
+	expect(t, recorder.Body.String(), "404 page not found\n")
+}
+
+func Test_RouterHandlerStatusCode(t *testing.T) {
+	router := NewRouter()
+	router.Get("/foo", func() string {
+		return "foo"
+	})
+	router.Get("/bar", func() (int, string) {
+		return http.StatusForbidden, "bar"
+	})
+	router.Get("/baz", func() (string, string) {
+		return "baz", "BAZ!"
+	})
+	router.Get("/bytes", func() []byte {
+		return []byte("Bytes!")
+	})
+	router.Get("/interface", func() interface{} {
+		return "Interface!"
+	})
+
+	// code should be 200 if none is returned from the handler
+	recorder := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "http://localhost:3000/foo", nil)
+	context := New().createContext(recorder, req)
+	router.Handle(recorder, req, context)
+	expect(t, recorder.Code, http.StatusOK)
+	expect(t, recorder.Body.String(), "foo")
+
+	// if a status code is returned, it should be used
+	recorder = httptest.NewRecorder()
+	req, _ = http.NewRequest("GET", "http://localhost:3000/bar", nil)
+	context = New().createContext(recorder, req)
+	router.Handle(recorder, req, context)
+	expect(t, recorder.Code, http.StatusForbidden)
+	expect(t, recorder.Body.String(), "bar")
+
+	// shouldn't use the first returned value as a status code if not an integer
+	recorder = httptest.NewRecorder()
+	req, _ = http.NewRequest("GET", "http://localhost:3000/baz", nil)
+	context = New().createContext(recorder, req)
+	router.Handle(recorder, req, context)
+	expect(t, recorder.Code, http.StatusOK)
+	expect(t, recorder.Body.String(), "baz")
+
+	// Should render bytes as a return value as well.
+	recorder = httptest.NewRecorder()
+	req, _ = http.NewRequest("GET", "http://localhost:3000/bytes", nil)
+	context = New().createContext(recorder, req)
+	router.Handle(recorder, req, context)
+	expect(t, recorder.Code, http.StatusOK)
+	expect(t, recorder.Body.String(), "Bytes!")
+
+	// Should render interface{} values.
+	recorder = httptest.NewRecorder()
+	req, _ = http.NewRequest("GET", "http://localhost:3000/interface", nil)
+	context = New().createContext(recorder, req)
+	router.Handle(recorder, req, context)
+	expect(t, recorder.Code, http.StatusOK)
+	expect(t, recorder.Body.String(), "Interface!")
+}
+
+func Test_RouterHandlerStacking(t *testing.T) {
+	router := NewRouter()
+	recorder := httptest.NewRecorder()
+
+	req, _ := http.NewRequest("GET", "http://localhost:3000/foo", nil)
+	context := New().createContext(recorder, req)
+
+	result := ""
+
+	f1 := func() {
+		result += "foo"
+	}
+
+	f2 := func(c Context) {
+		result += "bar"
+		c.Next()
+		result += "bing"
+	}
+
+	f3 := func() string {
+		result += "bat"
+		return "Hello world"
+	}
+
+	f4 := func() {
+		result += "baz"
+	}
+
+	router.Get("/foo", f1, f2, f3, f4)
+
+	router.Handle(recorder, req, context)
+	expect(t, result, "foobarbatbing")
+	expect(t, recorder.Body.String(), "Hello world")
+}
+
+var routeTests = []struct {
+	// in
+	method string
+	path   string
+
+	// out
+	ok     bool
+	params map[string]string
+}{
+	{"GET", "/foo/123/bat/321", true, map[string]string{"bar": "123", "baz": "321"}},
+	{"POST", "/foo/123/bat/321", false, map[string]string{}},
+	{"GET", "/foo/hello/bat/world", true, map[string]string{"bar": "hello", "baz": "world"}},
+	{"GET", "foo/hello/bat/world", false, map[string]string{}},
+	{"GET", "/foo/123/bat/321/", true, map[string]string{"bar": "123", "baz": "321"}},
+	{"GET", "/foo/123/bat/321//", false, map[string]string{}},
+	{"GET", "/foo/123//bat/321/", false, map[string]string{}},
+}
+
+func Test_RouteMatching(t *testing.T) {
+	route := newRoute("GET", "/foo/:bar/bat/:baz", nil)
+	for _, tt := range routeTests {
+		ok, params := route.Match(tt.method, tt.path)
+		if ok != tt.ok || params["bar"] != tt.params["bar"] || params["baz"] != tt.params["baz"] {
+			t.Errorf("expected: (%v, %v) got: (%v, %v)", tt.ok, tt.params, ok, params)
+		}
+	}
+}
+
+func Test_MethodsFor(t *testing.T) {
+	router := NewRouter()
+	recorder := httptest.NewRecorder()
+
+	req, _ := http.NewRequest("POST", "http://localhost:3000/foo", nil)
+	context := New().createContext(recorder, req)
+	context.MapTo(router, (*Routes)(nil))
+	router.Post("/foo/bar", func() {
+	})
+
+	router.Post("/fo", func() {
+	})
+
+	router.Get("/foo", func() {
+	})
+
+	router.Put("/foo", func() {
+	})
+
+	router.NotFound(func(routes Routes, w http.ResponseWriter, r *http.Request) {
+		methods := routes.MethodsFor(r.URL.Path)
+		if len(methods) != 0 {
+			w.Header().Set("Allow", strings.Join(methods, ","))
+			w.WriteHeader(http.StatusMethodNotAllowed)
+		}
+	})
+	router.Handle(recorder, req, context)
+	expect(t, recorder.Code, http.StatusMethodNotAllowed)
+	expect(t, recorder.Header().Get("Allow"), "GET,PUT")
+}
+
+func Test_NotFound(t *testing.T) {
+	router := NewRouter()
+	recorder := httptest.NewRecorder()
+
+	req, _ := http.NewRequest("GET", "http://localhost:3000/foo", nil)
+	context := New().createContext(recorder, req)
+
+	router.NotFound(func(res http.ResponseWriter) {
+		http.Error(res, "Nope", http.StatusNotFound)
+	})
+
+	router.Handle(recorder, req, context)
+	expect(t, recorder.Code, http.StatusNotFound)
+	expect(t, recorder.Body.String(), "Nope\n")
+}
+
+func Test_NotFoundAsHandler(t *testing.T) {
+	router := NewRouter()
+	recorder := httptest.NewRecorder()
+
+	req, _ := http.NewRequest("GET", "http://localhost:3000/foo", nil)
+	context := New().createContext(recorder, req)
+
+	router.NotFound(func() string {
+		return "not found"
+	})
+
+	router.Handle(recorder, req, context)
+	expect(t, recorder.Code, http.StatusOK)
+	expect(t, recorder.Body.String(), "not found")
+
+	recorder = httptest.NewRecorder()
+
+	context = New().createContext(recorder, req)
+
+	router.NotFound(func() (int, string) {
+		return 404, "not found"
+	})
+
+	router.Handle(recorder, req, context)
+	expect(t, recorder.Code, http.StatusNotFound)
+	expect(t, recorder.Body.String(), "not found")
+
+	recorder = httptest.NewRecorder()
+
+	context = New().createContext(recorder, req)
+
+	router.NotFound(func() (int, string) {
+		return 200, ""
+	})
+
+	router.Handle(recorder, req, context)
+	expect(t, recorder.Code, http.StatusOK)
+	expect(t, recorder.Body.String(), "")
+}
+
+func Test_NotFoundStacking(t *testing.T) {
+	router := NewRouter()
+	recorder := httptest.NewRecorder()
+
+	req, _ := http.NewRequest("GET", "http://localhost:3000/foo", nil)
+	context := New().createContext(recorder, req)
+
+	result := ""
+
+	f1 := func() {
+		result += "foo"
+	}
+
+	f2 := func(c Context) {
+		result += "bar"
+		c.Next()
+		result += "bing"
+	}
+
+	f3 := func() string {
+		result += "bat"
+		return "Not Found"
+	}
+
+	f4 := func() {
+		result += "baz"
+	}
+
+	router.NotFound(f1, f2, f3, f4)
+
+	router.Handle(recorder, req, context)
+	expect(t, result, "foobarbatbing")
+	expect(t, recorder.Body.String(), "Not Found")
+}
+
+func Test_Any(t *testing.T) {
+	router := NewRouter()
+	router.Any("/foo", func(res http.ResponseWriter) {
+		http.Error(res, "Nope", http.StatusNotFound)
+	})
+
+	recorder := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "http://localhost:3000/foo", nil)
+	context := New().createContext(recorder, req)
+	router.Handle(recorder, req, context)
+
+	expect(t, recorder.Code, http.StatusNotFound)
+	expect(t, recorder.Body.String(), "Nope\n")
+
+	recorder = httptest.NewRecorder()
+	req, _ = http.NewRequest("PUT", "http://localhost:3000/foo", nil)
+	context = New().createContext(recorder, req)
+	router.Handle(recorder, req, context)
+
+	expect(t, recorder.Code, http.StatusNotFound)
+	expect(t, recorder.Body.String(), "Nope\n")
+}
+
+func Test_URLFor(t *testing.T) {
+	router := NewRouter()
+
+	router.Get("/foo", func() {
+		// Nothing
+	}).Name("foo")
+
+	router.Post("/bar/:id", func(params Params) {
+		// Nothing
+	}).Name("bar")
+
+	router.Get("/bar/:id/:name", func(params Params, routes Routes) {
+		expect(t, routes.URLFor("foo", nil), "/foo")
+		expect(t, routes.URLFor("bar", 5), "/bar/5")
+		expect(t, routes.URLFor("bar_id", 5, "john"), "/bar/5/john")
+	}).Name("bar_id")
+
+	// code should be 200 if none is returned from the handler
+	recorder := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "http://localhost:3000/bar/foo/bar", nil)
+	context := New().createContext(recorder, req)
+	context.MapTo(router, (*Routes)(nil))
+	router.Handle(recorder, req, context)
+}

Godeps/_workspace/src/github.com/codegangsta/martini/static.go

@@ -0,0 +1,109 @@
+package martini
+
+import (
+	"log"
+	"net/http"
+	"path"
+	"strings"
+)
+
+// StaticOptions is a struct for specifying configuration options for the martini.Static middleware.
+type StaticOptions struct {
+	// Prefix is the optional prefix used to serve the static directory content
+	Prefix string
+	// SkipLogging will disable [Static] log messages when a static file is served.
+	SkipLogging bool
+	// IndexFile defines which file to serve as index if it exists.
+	IndexFile string
+	// Expires defines which user-defined function to use for producing a HTTP Expires Header
+	// https://developers.google.com/speed/docs/insights/LeverageBrowserCaching
+	Expires func() string
+}
+
+func prepareStaticOptions(options []StaticOptions) StaticOptions {
+	var opt StaticOptions
+	if len(options) > 0 {
+		opt = options[0]
+	}
+
+	// Defaults
+	if len(opt.IndexFile) == 0 {
+		opt.IndexFile = "index.html"
+	}
+	// Normalize the prefix if provided
+	if opt.Prefix != "" {
+		// Ensure we have a leading '/'
+		if opt.Prefix[0] != '/' {
+			opt.Prefix = "/" + opt.Prefix
+		}
+		// Remove any trailing '/'
+		opt.Prefix = strings.TrimRight(opt.Prefix, "/")
+	}
+	return opt
+}
+
+// Static returns a middleware handler that serves static files in the given directory.
+func Static(directory string, staticOpt ...StaticOptions) Handler {
+	dir := http.Dir(directory)
+	opt := prepareStaticOptions(staticOpt)
+
+	return func(res http.ResponseWriter, req *http.Request, log *log.Logger) {
+		if req.Method != "GET" && req.Method != "HEAD" {
+			return
+		}
+		file := req.URL.Path
+		// if we have a prefix, filter requests by stripping the prefix
+		if opt.Prefix != "" {
+			if !strings.HasPrefix(file, opt.Prefix) {
+				return
+			}
+			file = file[len(opt.Prefix):]
+			if file != "" && file[0] != '/' {
+				return
+			}
+		}
+		f, err := dir.Open(file)
+		if err != nil {
+			// discard the error?
+			return
+		}
+		defer f.Close()
+
+		fi, err := f.Stat()
+		if err != nil {
+			return
+		}
+
+		// try to serve index file
+		if fi.IsDir() {
+			// redirect if missing trailing slash
+			if !strings.HasSuffix(req.URL.Path, "/") {
+				http.Redirect(res, req, req.URL.Path+"/", http.StatusFound)
+				return
+			}
+
+			file = path.Join(file, opt.IndexFile)
+			f, err = dir.Open(file)
+			if err != nil {
+				return
+			}
+			defer f.Close()
+
+			fi, err = f.Stat()
+			if err != nil || fi.IsDir() {
+				return
+			}
+		}
+
+		if !opt.SkipLogging {
+			log.Println("[Static] Serving " + file)
+		}
+
+		// Add an Expires header to the static content
+		if opt.Expires != nil {
+			res.Header().Set("Expires", opt.Expires())
+		}
+
+		http.ServeContent(res, req, file, fi.ModTime(), f)
+	}
+}

Godeps/_workspace/src/github.com/codegangsta/martini/static_test.go

@@ -0,0 +1,200 @@
+package martini
+
+import (
+	"bytes"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/codegangsta/inject"
+)
+
+func Test_Static(t *testing.T) {
+	response := httptest.NewRecorder()
+	response.Body = new(bytes.Buffer)
+
+	m := New()
+	r := NewRouter()
+
+	m.Use(Static("."))
+	m.Action(r.Handle)
+
+	req, err := http.NewRequest("GET", "http://localhost:3000/martini.go", nil)
+	if err != nil {
+		t.Error(err)
+	}
+	m.ServeHTTP(response, req)
+	expect(t, response.Code, http.StatusOK)
+	expect(t, response.Header().Get("Expires"), "")
+	if response.Body.Len() == 0 {
+		t.Errorf("Got empty body for GET request")
+	}
+}
+
+func Test_Static_Head(t *testing.T) {
+	response := httptest.NewRecorder()
+	response.Body = new(bytes.Buffer)
+
+	m := New()
+	r := NewRouter()
+
+	m.Use(Static("."))
+	m.Action(r.Handle)
+
+	req, err := http.NewRequest("HEAD", "http://localhost:3000/martini.go", nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	m.ServeHTTP(response, req)
+	expect(t, response.Code, http.StatusOK)
+	if response.Body.Len() != 0 {
+		t.Errorf("Got non-empty body for HEAD request")
+	}
+}
+
+func Test_Static_As_Post(t *testing.T) {
+	response := httptest.NewRecorder()
+
+	m := New()
+	r := NewRouter()
+
+	m.Use(Static("."))
+	m.Action(r.Handle)
+
+	req, err := http.NewRequest("POST", "http://localhost:3000/martini.go", nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	m.ServeHTTP(response, req)
+	expect(t, response.Code, http.StatusNotFound)
+}
+
+func Test_Static_BadDir(t *testing.T) {
+	response := httptest.NewRecorder()
+
+	m := Classic()
+
+	req, err := http.NewRequest("GET", "http://localhost:3000/martini.go", nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	m.ServeHTTP(response, req)
+	refute(t, response.Code, http.StatusOK)
+}
+
+func Test_Static_Options_Logging(t *testing.T) {
+	response := httptest.NewRecorder()
+
+	var buffer bytes.Buffer
+	m := &Martini{Injector: inject.New(), action: func() {}, logger: log.New(&buffer, "[martini] ", 0)}
+	m.Map(m.logger)
+	m.Map(defaultReturnHandler())
+
+	opt := StaticOptions{}
+	m.Use(Static(".", opt))
+
+	req, err := http.NewRequest("GET", "http://localhost:3000/martini.go", nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	m.ServeHTTP(response, req)
+	expect(t, response.Code, http.StatusOK)
+	expect(t, buffer.String(), "[martini] [Static] Serving /martini.go\n")
+
+	// Now without logging
+	m.Handlers()
+	buffer.Reset()
+
+	// This should disable logging
+	opt.SkipLogging = true
+	m.Use(Static(".", opt))
+
+	m.ServeHTTP(response, req)
+	expect(t, response.Code, http.StatusOK)
+	expect(t, buffer.String(), "")
+}
+
+func Test_Static_Options_ServeIndex(t *testing.T) {
+	response := httptest.NewRecorder()
+
+	var buffer bytes.Buffer
+	m := &Martini{Injector: inject.New(), action: func() {}, logger: log.New(&buffer, "[martini] ", 0)}
+	m.Map(m.logger)
+	m.Map(defaultReturnHandler())
+
+	opt := StaticOptions{IndexFile: "martini.go"} // Define martini.go as index file
+	m.Use(Static(".", opt))
+
+	req, err := http.NewRequest("GET", "http://localhost:3000/", nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	m.ServeHTTP(response, req)
+	expect(t, response.Code, http.StatusOK)
+	expect(t, buffer.String(), "[martini] [Static] Serving /martini.go\n")
+}
+
+func Test_Static_Options_Prefix(t *testing.T) {
+	response := httptest.NewRecorder()
+
+	var buffer bytes.Buffer
+	m := &Martini{Injector: inject.New(), action: func() {}, logger: log.New(&buffer, "[martini] ", 0)}
+	m.Map(m.logger)
+	m.Map(defaultReturnHandler())
+
+	// Serve current directory under /public
+	m.Use(Static(".", StaticOptions{Prefix: "/public"}))
+
+	// Check file content behaviour
+	req, err := http.NewRequest("GET", "http://localhost:3000/public/martini.go", nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	m.ServeHTTP(response, req)
+	expect(t, response.Code, http.StatusOK)
+	expect(t, buffer.String(), "[martini] [Static] Serving /martini.go\n")
+}
+
+func Test_Static_Options_Expires(t *testing.T) {
+	response := httptest.NewRecorder()
+
+	var buffer bytes.Buffer
+	m := &Martini{Injector: inject.New(), action: func() {}, logger: log.New(&buffer, "[martini] ", 0)}
+	m.Map(m.logger)
+	m.Map(defaultReturnHandler())
+
+	// Serve current directory under /public
+	m.Use(Static(".", StaticOptions{Expires: func() string { return "46" }}))
+
+	// Check file content behaviour
+	req, err := http.NewRequest("GET", "http://localhost:3000/martini.go", nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	m.ServeHTTP(response, req)
+	expect(t, response.Header().Get("Expires"), "46")
+}
+
+func Test_Static_Redirect(t *testing.T) {
+	response := httptest.NewRecorder()
+
+	m := New()
+	m.Use(Static(".", StaticOptions{Prefix: "/public"}))
+
+	req, err := http.NewRequest("GET", "http://localhost:3000/public", nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	m.ServeHTTP(response, req)
+	expect(t, response.Code, http.StatusFound)
+	expect(t, response.Header().Get("Location"), "/public/")
+}

Godeps/_workspace/src/github.com/codegangsta/martini/translations/README_zh_cn.md

@@ -0,0 +1,311 @@
+# Martini  [![wercker status](https://app.wercker.com/status/174bef7e3c999e103cacfe2770102266 "wercker status")](https://app.wercker.com/project/bykey/174bef7e3c999e103cacfe2770102266) [![GoDoc](https://godoc.org/github.com/codegangsta/martini?status.png)](http://godoc.org/github.com/codegangsta/martini)
+
+Martini是一个强大为了编写模块化Web应用而生的GO语言框架.
+
+## 第一个应用
+
+在你安装了GO语言和设置了你的[GOPATH](http://golang.org/doc/code.html#GOPATH)之后, 创建你的自己的`.go`文件, 这里我们假设它的名字叫做 `server.go`.
+
+~~~ go
+package main
+
+import "github.com/codegangsta/martini"
+
+func main() {
+  m := martini.Classic()
+  m.Get("/", func() string {
+    return "Hello world!"
+  })
+  m.Run()
+}
+~~~
+
+然后安装Martini的包. (注意Martini需要Go语言1.1或者以上的版本支持):
+~~~
+go get github.com/codegangsta/martini
+~~~
+
+最后运行你的服务:
+~~~
+go run server.go
+~~~
+
+这时你将会有一个Martini的服务监听了, 地址是: `localhost:3000`.
+
+## 获得帮助
+
+请加入: [邮件列表](https://groups.google.com/forum/#!forum/martini-go)
+
+或者可以查看在线演示地址: [演示视频](http://martini.codegangsta.io/#demo)
+
+## 功能列表
+* 使用极其简单.
+* 无侵入式的设计.
+* 很好的与其他的Go语言包协同使用.
+* 超赞的路径匹配和路由.
+* 模块化的设计 - 容易插入功能件，也容易将其拔出来.
+* 已有很多的中间件可以直接使用.
+* 框架内已拥有很好的开箱即用的功能支持.
+* **完全兼容[http.HandlerFunc](http://godoc.org/net/http#HandlerFunc)接口.**
+
+## 更多中间件
+更多的中间件和功能组件, 请查看代码仓库: [martini-contrib](https://github.com/martini-contrib).
+
+## 目录
+* [核心 Martini](#classic-martini)
+  * [处理器](#handlers)
+  * [路由](#routing)
+  * [服务](#services)
+  * [服务静态文件](#serving-static-files)
+* [中间件处理器](#middleware-handlers)
+  * [Next()](#next)
+* [常见问答](#faq)
+
+## 核心 Martini
+为了更快速的启用Martini, [martini.Classic()](http://godoc.org/github.com/codegangsta/martini#Classic) 提供了一些默认的方便Web开发的工具:
+~~~ go
+  m := martini.Classic()
+  // ... middleware and routing goes here
+  m.Run()
+~~~
+
+下面是Martini核心已经包含的功能 [martini.Classic()](http://godoc.org/github.com/codegangsta/martini#Classic):
+  * Request/Response Logging （请求/相应日志） - [martini.Logger](http://godoc.org/github.com/codegangsta/martini#Logger)
+  * Panic Recovery （容错） - [martini.Recovery](http://godoc.org/github.com/codegangsta/martini#Recovery)
+  * Static File serving （静态文件服务） - [martini.Static](http://godoc.org/github.com/codegangsta/martini#Static)
+  * Routing （路由） - [martini.Router](http://godoc.org/github.com/codegangsta/martini#Router)
+
+### 处理器
+处理器是Martini的灵魂和核心所在. 一个处理器基本上可以是任何的函数:
+~~~ go
+m.Get("/", func() {
+  println("hello world")
+})
+~~~
+
+#### 返回值
+当一个处理器返回结果的时候, Martini将会把返回值作为字符串写入到当前的[http.ResponseWriter](http://godoc.org/net/http#ResponseWriter)里面:
+~~~ go
+m.Get("/", func() string {
+  return "hello world" // HTTP 200 : "hello world"
+})
+~~~
+
+另外你也可以选择性的返回多一个状态码:
+~~~ go
+m.Get("/", func() (int, string) {
+  return 418, "i'm a teapot" // HTTP 418 : "i'm a teapot"
+})
+~~~
+
+#### 服务的注入 
+处理器是通过反射来调用的. Martini 通过*Dependency Injection* *（依赖注入）* 来为处理器注入参数列表. **这样使得Martini与Go语言的`http.HandlerFunc`接口完全兼容.** 
+
+如果你加入一个参数到你的处理器, Martini将会搜索它参数列表中的服务，并且通过类型判断来解决依赖关系:
+~~~ go
+m.Get("/", func(res http.ResponseWriter, req *http.Request) { // res 和 req 是通过Martini注入的
+  res.WriteHeader(200) // HTTP 200
+})
+~~~
+
+下面的这些服务已经被包含在核心Martini中: [martini.Classic()](http://godoc.org/github.com/codegangsta/martini#Classic):
+  * [*log.Logger](http://godoc.org/log#Logger) - Martini的全局日志.
+  * [martini.Context](http://godoc.org/github.com/codegangsta/martini#Context) - http request context （请求上下文）.
+  * [martini.Params](http://godoc.org/github.com/codegangsta/martini#Params) - `map[string]string` of named params found by route matching. （名字和参数键值对的参数列表）
+  * [martini.Routes](http://godoc.org/github.com/codegangsta/martini#Routes) - Route helper service. （路由协助处理）
+  * [http.ResponseWriter](http://godoc.org/net/http/#ResponseWriter) - http Response writer interface. (响应结果的流接口)
+  * [*http.Request](http://godoc.org/net/http/#Request) - http Request. （http请求)
+
+### 路由
+在Martini中, 路由是一个HTTP方法配对一个URL匹配模型. 每一个路由可以对应一个或多个处理器方法:
+~~~ go
+m.Get("/", func() {
+  // 显示
+})
+
+m.Patch("/", func() {
+  // 更新
+})
+
+m.Post("/", func() {
+  // 创建
+})
+
+m.Put("/", func() {
+  // 替换
+})
+
+m.Delete("/", func() {
+  // 删除
+})
+
+m.Options("/", func() {
+  // http 选项
+})
+
+m.NotFound(func() {
+  // 处理 404
+})
+~~~
+
+路由匹配的顺序是按照他们被定义的顺序执行的. 最先被定义的路由将会首先被用户请求匹配并调用.
+
+路由模型可能包含参数列表, 可以通过[martini.Params](http://godoc.org/github.com/codegangsta/martini#Params)服务来获取:
+~~~ go
+m.Get("/hello/:name", func(params martini.Params) string {
+  return "Hello " + params["name"]
+})
+~~~
+
+路由匹配可以通过正则表达式或者glob的形式:
+~~~ go
+m.Get("/hello/**", func(params martini.Params) string {
+  return "Hello " + params["_1"]
+})
+~~~
+
+路由处理器可以被相互叠加使用, 例如很有用的地方可以是在验证和授权的时候:
+~~~ go
+m.Get("/secret", authorize, func() {
+  // 该方法将会在authorize方法没有输出结果的时候执行.
+})
+~~~
+
+### 服务
+服务即是被注入到处理器中的参数. 你可以映射一个服务到 *全局* 或者 *请求* 的级别.
+
+
+#### 全局映射
+如果一个Martini实现了inject.Injector的接口, 那么映射成为一个服务就非常简单:
+~~~ go
+db := &MyDatabase{}
+m := martini.Classic()
+m.Map(db) // *MyDatabase 这个服务将可以在所有的处理器中被使用到.
+// ...
+m.Run()
+~~~
+
+#### 请求级别的映射
+映射在请求级别的服务可以用[martini.Context](http://godoc.org/github.com/codegangsta/martini#Context)来完成:
+~~~ go
+func MyCustomLoggerHandler(c martini.Context, req *http.Request) {
+  logger := &MyCustomLogger{req}
+  c.Map(logger) // 映射成为了 *MyCustomLogger
+}
+~~~
+
+#### 映射值到接口
+关于服务最强悍的地方之一就是它能够映射服务到接口. 例如说, 假设你想要覆盖[http.ResponseWriter](http://godoc.org/net/http#ResponseWriter)成为一个对象, 那么你可以封装它并包含你自己的额外操作, 你可以如下这样来编写你的处理器:
+~~~ go
+func WrapResponseWriter(res http.ResponseWriter, c martini.Context) {
+  rw := NewSpecialResponseWriter(res)
+  c.MapTo(rw, (*http.ResponseWriter)(nil)) // 覆盖 ResponseWriter 成为我们封装过的 ResponseWriter
+}
+~~~
+
+### 服务静态文件
+[martini.Classic()](http://godoc.org/github.com/codegangsta/martini#Classic) 默认会服务位于你服务器环境根目录下的"public"文件夹.
+你可以通过加入[martini.Static](http://godoc.org/github.com/codegangsta/martini#Static)的处理器来加入更多的静态文件服务的文件夹.
+~~~ go
+m.Use(martini.Static("assets")) // 也会服务静态文件于"assets"的文件夹
+~~~
+
+## 中间件处理器
+中间件处理器是工作于请求和路由之间的. 本质上来说和Martini其他的处理器没有分别. 你可以像如下这样添加一个中间件处理器到它的堆中:
+~~~ go
+m.Use(func() {
+  // 做一些中间件该做的事情
+})
+~~~
+
+你可以通过`Handlers`函数对中间件堆有完全的控制. 它将会替换掉之前的任何设置过的处理器:
+~~~ go
+m.Handlers(
+  Middleware1,
+  Middleware2,
+  Middleware3,
+)
+~~~
+
+中间件处理器可以非常好处理一些功能，像logging(日志), authorization(授权), authentication(认证), sessions(会话), error pages(错误页面), 以及任何其他的操作需要在http请求发生之前或者之后的:
+
+~~~ go
+// 验证api密匙
+m.Use(func(res http.ResponseWriter, req *http.Request) {
+  if req.Header.Get("X-API-KEY") != "secret123" {
+    res.WriteHeader(http.StatusUnauthorized)
+  }
+})
+~~~
+
+### Next()
+[Context.Next()](http://godoc.org/github.com/codegangsta/martini#Context)是一个可选的函数用于中间件处理器暂时放弃执行直到其他的处理器都执行完毕. 这样就可以很好的处理在http请求完成后需要做的操作.
+~~~ go
+// log 记录请求完成前后  (*译者注: 很巧妙，掌声鼓励.)
+m.Use(func(c martini.Context, log *log.Logger){
+  log.Println("before a request")
+
+  c.Next()
+  
+  log.Println("after a request")
+})
+~~~
+
+## 常见问答
+
+### 我在哪里可以找到中间件资源?
+
+可以查看 [martini-contrib](https://github.com/martini-contrib) 项目. 如果看了觉得没有什么好货色, 可以联系martini-contrib的团队成员为你创建一个新的代码资源库.
+
+* [auth](https://github.com/martini-contrib/auth) - 认证处理器.
+* [binding](https://github.com/martini-contrib/binding) - 映射/验证raw请求到结构体(structure)里的处理器
+* [gzip](https://github.com/martini-contrib/gzip) - 加入giz支持的处理器
+* [render](https://github.com/martini-contrib/render) - 渲染JSON和HTML模板的处理器.
+* [acceptlang](https://github.com/martini-contrib/acceptlang) - 解析`Accept-Language` HTTP报头的处理器.
+* [sessions](https://github.com/martini-contrib/sessions) - 提供会话服务支持的处理器.
+* [strip](https://github.com/martini-contrib/strip) - URL Prefix stripping.
+* [method](https://github.com/martini-contrib/method) - HTTP method overriding via Header or form fields.
+* [secure](https://github.com/martini-contrib/secure) - Implements a few quick security wins.
+* [encoder](https://github.com/martini-contrib/encoder) - Encoder service for rendering data in several formats and content negotiation.
+
+### 我如何整合到我现有的服务器中?
+
+由于Martini实现了 `http.Handler`, 所以它可以很简单的应用到现有Go服务器的子集中. 例如说这是一段在Google App Engine中的示例:
+
+~~~ go
+package hello
+
+import (
+  "net/http"
+  "github.com/codegangsta/martini"
+)
+
+func init() {
+  m := martini.Classic()
+  m.Get("/", func() string {
+    return "Hello world!"
+  })
+  http.Handle("/", m)
+}
+~~~
+
+### 我如何修改port/host?
+
+Martini的`Run`函数会检查PORT和HOST的环境变量并使用它们. 否则Martini将会默认使用localhost:3000
+如果想要自定义PORT和HOST, 使用`http.ListenAndServe`函数来代替.
+
+~~~ go
+  m := martini.Classic()
+  // ...
+  http.ListenAndServe(":8080", m)
+~~~
+
+## 贡献
+Martini项目想要保持简单且干净的代码. 大部分的代码应该贡献到[martini-contrib](https://github.com/martini-contrib)组织中作为一个项目. 如果你想要贡献Martini的核心代码也可以发起一个Pull Request.
+
+## 关于
+
+灵感来自于 [express](https://github.com/visionmedia/express) 和 [sinatra](https://github.com/sinatra/sinatra)
+
+Martini作者 [Code Gangsta](http://codegangsta.io/)
+译者: [Leon](http://github.com/leonli)

Godeps/_workspace/src/github.com/codegangsta/martini/wercker.yml

@@ -0,0 +1 @@
+box: wercker/golang@1.1.1

Godeps/_workspace/src/github.com/juju/ratelimit/LICENSE

@@ -0,0 +1,185 @@
+This software is licensed under the LGPLv3, included below.
+
+As a special exception to the GNU Lesser General Public License version 3
+("LGPL3"), the copyright holders of this Library give you permission to
+convey to a third party a Combined Work that links statically or dynamically
+to this Library without providing any Minimal Corresponding Source or
+Minimal Application Code as set out in 4d or providing the installation
+information set out in section 4e, provided that you comply with the other
+provisions of LGPL3 and provided that you meet, for the Application the
+terms and conditions of the license(s) which apply to the Application.
+
+Except as stated in this special exception, the provisions of LGPL3 will
+continue to comply in full to this Library. If you modify this Library, you
+may apply this exception to your version of this Library, but you are not
+obliged to do so. If you do not wish to do so, delete this exception
+statement from your version. This exception does not (and cannot) modify any
+license terms which apply to the Application, with which you must still
+comply.
+
+
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

Godeps/_workspace/src/github.com/juju/ratelimit/README.md

@@ -0,0 +1,109 @@
+# ratelimit
+--
+    import "github.com/juju/ratelimit"
+
+The ratelimit package provides an efficient token bucket implementation. See
+http://en.wikipedia.org/wiki/Token_bucket.
+
+## Usage
+
+#### func  Reader
+
+```go
+func Reader(r io.Reader, bucket *Bucket) io.Reader
+```
+Reader returns a reader that is rate limited by the given token bucket. Each
+token in the bucket represents one byte.
+
+#### func  Writer
+
+```go
+func Writer(w io.Writer, bucket *Bucket) io.Writer
+```
+Writer returns a reader that is rate limited by the given token bucket. Each
+token in the bucket represents one byte.
+
+#### type Bucket
+
+```go
+type Bucket struct {
+}
+```
+
+Bucket represents a token bucket that fills at a predetermined rate. Methods on
+Bucket may be called concurrently.
+
+#### func  NewBucket
+
+```go
+func NewBucket(fillInterval time.Duration, capacity int64) *Bucket
+```
+NewBucket returns a new token bucket that fills at the rate of one token every
+fillInterval, up to the given maximum capacity. Both arguments must be positive.
+The bucket is initially full.
+
+#### func  NewBucketWithRate
+
+```go
+func NewBucketWithRate(rate float64, capacity int64) *Bucket
+```
+NewBucketWithRate returns a token bucket that fills the bucket at the rate of
+rate tokens per second up to the given maximum capacity. Because of limited
+clock resolution, at high rates, the actual rate may be up to 1% different from
+the specified rate.
+
+#### func (*Bucket) Rate
+
+```go
+func (tb *Bucket) Rate() float64
+```
+Rate returns the fill rate of the bucket, in tokens per second.
+
+#### func (*Bucket) Take
+
+```go
+func (tb *Bucket) Take(count int64) time.Duration
+```
+Take takes count tokens from the bucket without blocking. It returns the time
+that the caller should wait until the tokens are actually available.
+
+Note that if the request is irrevocable - there is no way to return tokens to
+the bucket once this method commits us to taking them.
+
+#### func (*Bucket) TakeAvailable
+
+```go
+func (tb *Bucket) TakeAvailable(count int64) int64
+```
+TakeAvailable takes up to count immediately available tokens from the bucket. It
+returns the number of tokens removed, or zero if there are no available tokens.
+It does not block.
+
+#### func (*Bucket) TakeMaxDuration
+
+```go
+func (tb *Bucket) TakeMaxDuration(count int64, maxWait time.Duration) (time.Duration, bool)
+```
+TakeMaxDuration is like Take, except that it will only take tokens from the
+bucket if the wait time for the tokens is no greater than maxWait.
+
+If it would take longer than maxWait for the tokens to become available, it does
+nothing and reports false, otherwise it returns the time that the caller should
+wait until the tokens are actually available, and reports true.
+
+#### func (*Bucket) Wait
+
+```go
+func (tb *Bucket) Wait(count int64)
+```
+Wait takes count tokens from the bucket, waiting until they are available.
+
+#### func (*Bucket) WaitMaxDuration
+
+```go
+func (tb *Bucket) WaitMaxDuration(count int64, maxWait time.Duration) bool
+```
+WaitMaxDuration is like Wait except that it will only take tokens from the
+bucket if it needs to wait for no greater than maxWait. It reports whether any
+tokens have been removed from the bucket If no tokens have been removed, it
+returns immediately.

Godeps/_workspace/src/github.com/juju/ratelimit/ratelimit.go

@@ -0,0 +1,227 @@
+// Copyright 2014 Canonical Ltd.
+// Licensed under the LGPLv3 with static-linking exception.
+// See LICENCE file for details.
+
+// The ratelimit package provides an efficient token bucket implementation.
+// See http://en.wikipedia.org/wiki/Token_bucket.
+package ratelimit
+
+import (
+	"strconv"
+	"sync"
+	"time"
+)
+
+// Bucket represents a token bucket that fills at a predetermined rate.
+// Methods on Bucket may be called concurrently.
+type Bucket struct {
+	startTime    time.Time
+	capacity     int64
+	quantum      int64
+	fillInterval time.Duration
+
+	// The mutex guards the fields following it.
+	mu sync.Mutex
+
+	// avail holds the number of available tokens
+	// in the bucket, as of availTick ticks from startTime.
+	// It will be negative when there are consumers
+	// waiting for tokens.
+	avail     int64
+	availTick int64
+}
+
+// NewBucket returns a new token bucket that fills at the
+// rate of one token every fillInterval, up to the given
+// maximum capacity. Both arguments must be
+// positive. The bucket is initially full.
+func NewBucket(fillInterval time.Duration, capacity int64) *Bucket {
+	return newBucketWithQuantum(fillInterval, capacity, 1)
+}
+
+// rateMargin specifes the allowed variance of actual
+// rate from specified rate. 1% seems reasonable.
+const rateMargin = 0.01
+
+// NewBucketWithRate returns a token bucket that fills the bucket
+// at the rate of rate tokens per second up to the given
+// maximum capacity. Because of limited clock resolution,
+// at high rates, the actual rate may be up to 1% different from the
+// specified rate.
+func NewBucketWithRate(rate float64, capacity int64) *Bucket {
+	for quantum := int64(1); quantum < 1<<50; quantum = nextQuantum(quantum) {
+		fillInterval := time.Duration(1e9 * float64(quantum) / rate)
+		if fillInterval <= 0 {
+			continue
+		}
+		tb := newBucketWithQuantum(fillInterval, capacity, quantum)
+		if diff := abs(tb.Rate() - rate); diff/rate <= rateMargin {
+			return tb
+		}
+	}
+	panic("cannot find suitable quantum for " + strconv.FormatFloat(rate, 'g', -1, 64))
+}
+
+// nextQuantum returns the next quantum to try after q.
+// We grow the quantum exponentially, but slowly, so we
+// get a good fit in the lower numbers.
+func nextQuantum(q int64) int64 {
+	q1 := q * 11 / 10
+	if q1 == q {
+		q1++
+	}
+	return q1
+}
+
+// newBucketWithQuantum is similar to NewBucket, but allows
+// the specification of the quantum size - quantum tokens
+// are added every fillInterval. This is so that we can get accurate
+// rates even when we want to add more than one token per ns.
+func newBucketWithQuantum(fillInterval time.Duration, capacity, quantum int64) *Bucket {
+	if fillInterval <= 0 {
+		panic("token bucket fill interval is not > 0")
+	}
+	if capacity <= 0 {
+		panic("token bucket capacity is not > 0")
+	}
+	if quantum <= 0 {
+		panic("token bucket quantum is not > 0")
+	}
+	return &Bucket{
+		startTime:    time.Now(),
+		capacity:     capacity,
+		quantum:      quantum,
+		avail:        capacity,
+		fillInterval: fillInterval,
+	}
+}
+
+// Wait takes count tokens from the bucket, waiting until they are
+// available.
+func (tb *Bucket) Wait(count int64) {
+	if d := tb.Take(count); d > 0 {
+		time.Sleep(d)
+	}
+}
+
+// WaitMaxDuration is like Wait except that it will
+// only take tokens from the bucket if it needs to wait
+// for no greater than maxWait. It reports whether
+// any tokens have been removed from the bucket
+// If no tokens have been removed, it returns immediately.
+func (tb *Bucket) WaitMaxDuration(count int64, maxWait time.Duration) bool {
+	d, ok := tb.TakeMaxDuration(count, maxWait)
+	if d > 0 {
+		time.Sleep(d)
+	}
+	return ok
+}
+
+const infinityDuration time.Duration = 0x7fffffffffffffff
+
+// Take takes count tokens from the bucket without blocking. It returns
+// the time that the caller should wait until the tokens are actually
+// available.
+//
+// Note that if the request is irrevocable - there is no way to return
+// tokens to the bucket once this method commits us to taking them.
+func (tb *Bucket) Take(count int64) time.Duration {
+	d, _ := tb.take(time.Now(), count, infinityDuration)
+	return d
+}
+
+// TakeMaxDuration is like Take, except that
+// it will only take tokens from the bucket if the wait
+// time for the tokens is no greater than maxWait.
+//
+// If it would take longer than maxWait for the tokens
+// to become available, it does nothing and reports false,
+// otherwise it returns the time that the caller should
+// wait until the tokens are actually available, and reports
+// true.
+func (tb *Bucket) TakeMaxDuration(count int64, maxWait time.Duration) (time.Duration, bool) {
+	return tb.take(time.Now(), count, maxWait)
+}
+
+// TakeAvailable takes up to count immediately available tokens from the
+// bucket. It returns the number of tokens removed, or zero if there are
+// no available tokens. It does not block.
+func (tb *Bucket) TakeAvailable(count int64) int64 {
+	return tb.takeAvailable(time.Now(), count)
+}
+
+// takeAvailable is the internal version of TakeAvailable - it takes the
+// current time as an argument to enable easy testing.
+func (tb *Bucket) takeAvailable(now time.Time, count int64) int64 {
+	if count <= 0 {
+		return 0
+	}
+	tb.mu.Lock()
+	defer tb.mu.Unlock()
+
+	tb.adjust(now)
+	if tb.avail <= 0 {
+		return 0
+	}
+	if count > tb.avail {
+		count = tb.avail
+	}
+	tb.avail -= count
+	return count
+}
+
+// Rate returns the fill rate of the bucket, in tokens per second.
+func (tb *Bucket) Rate() float64 {
+	return 1e9 * float64(tb.quantum) / float64(tb.fillInterval)
+}
+
+// take is the internal version of Take - it takes the current time as
+// an argument to enable easy testing.
+func (tb *Bucket) take(now time.Time, count int64, maxWait time.Duration) (time.Duration, bool) {
+	if count <= 0 {
+		return 0, true
+	}
+	tb.mu.Lock()
+	defer tb.mu.Unlock()
+
+	currentTick := tb.adjust(now)
+	avail := tb.avail - count
+	if avail >= 0 {
+		tb.avail = avail
+		return 0, true
+	}
+	// Round up the missing tokens to the nearest multiple
+	// of quantum - the tokens won't be available until
+	// that tick.
+	endTick := currentTick + (-avail+tb.quantum-1)/tb.quantum
+	endTime := tb.startTime.Add(time.Duration(endTick) * tb.fillInterval)
+	waitTime := endTime.Sub(now)
+	if waitTime > maxWait {
+		return 0, false
+	}
+	tb.avail = avail
+	return waitTime, true
+}
+
+// adjust adjusts the current bucket capacity based on the current time.
+// It returns the current tick.
+func (tb *Bucket) adjust(now time.Time) (currentTick int64) {
+	currentTick = int64(now.Sub(tb.startTime) / tb.fillInterval)
+
+	if tb.avail >= tb.capacity {
+		return
+	}
+	tb.avail += (currentTick - tb.availTick) * tb.quantum
+	if tb.avail > tb.capacity {
+		tb.avail = tb.capacity
+	}
+	tb.availTick = currentTick
+	return
+}
+
+func abs(f float64) float64 {
+	if f < 0 {
+		return -f
+	}
+	return f
+}

Godeps/_workspace/src/github.com/juju/ratelimit/ratelimit_test.go

@@ -0,0 +1,328 @@
+// Copyright 2014 Canonical Ltd.
+// Licensed under the LGPLv3 with static-linking exception.
+// See LICENCE file for details.
+
+package ratelimit
+
+import (
+	gc "launchpad.net/gocheck"
+
+	"testing"
+	"time"
+)
+
+func TestPackage(t *testing.T) {
+	gc.TestingT(t)
+}
+
+type rateLimitSuite struct{}
+
+var _ = gc.Suite(rateLimitSuite{})
+
+type takeReq struct {
+	time       time.Duration
+	count      int64
+	expectWait time.Duration
+}
+
+var takeTests = []struct {
+	about        string
+	fillInterval time.Duration
+	capacity     int64
+	reqs         []takeReq
+}{{
+	about:        "serial requests",
+	fillInterval: 250 * time.Millisecond,
+	capacity:     10,
+	reqs: []takeReq{{
+		time:       0,
+		count:      0,
+		expectWait: 0,
+	}, {
+		time:       0,
+		count:      10,
+		expectWait: 0,
+	}, {
+		time:       0,
+		count:      1,
+		expectWait: 250 * time.Millisecond,
+	}, {
+		time:       250 * time.Millisecond,
+		count:      1,
+		expectWait: 250 * time.Millisecond,
+	}},
+}, {
+	about:        "concurrent requests",
+	fillInterval: 250 * time.Millisecond,
+	capacity:     10,
+	reqs: []takeReq{{
+		time:       0,
+		count:      10,
+		expectWait: 0,
+	}, {
+		time:       0,
+		count:      2,
+		expectWait: 500 * time.Millisecond,
+	}, {
+		time:       0,
+		count:      2,
+		expectWait: 1000 * time.Millisecond,
+	}, {
+		time:       0,
+		count:      1,
+		expectWait: 1250 * time.Millisecond,
+	}},
+}, {
+	about:        "more than capacity",
+	fillInterval: 1 * time.Millisecond,
+	capacity:     10,
+	reqs: []takeReq{{
+		time:       0,
+		count:      10,
+		expectWait: 0,
+	}, {
+		time:       20 * time.Millisecond,
+		count:      15,
+		expectWait: 5 * time.Millisecond,
+	}},
+}, {
+	about:        "sub-quantum time",
+	fillInterval: 10 * time.Millisecond,
+	capacity:     10,
+	reqs: []takeReq{{
+		time:       0,
+		count:      10,
+		expectWait: 0,
+	}, {
+		time:       7 * time.Millisecond,
+		count:      1,
+		expectWait: 3 * time.Millisecond,
+	}, {
+		time:       8 * time.Millisecond,
+		count:      1,
+		expectWait: 12 * time.Millisecond,
+	}},
+}, {
+	about:        "within capacity",
+	fillInterval: 10 * time.Millisecond,
+	capacity:     5,
+	reqs: []takeReq{{
+		time:       0,
+		count:      5,
+		expectWait: 0,
+	}, {
+		time:       60 * time.Millisecond,
+		count:      5,
+		expectWait: 0,
+	}, {
+		time:       60 * time.Millisecond,
+		count:      1,
+		expectWait: 10 * time.Millisecond,
+	}, {
+		time:       80 * time.Millisecond,
+		count:      2,
+		expectWait: 10 * time.Millisecond,
+	}},
+}}
+
+func (rateLimitSuite) TestTake(c *gc.C) {
+	for i, test := range takeTests {
+		tb := NewBucket(test.fillInterval, test.capacity)
+		for j, req := range test.reqs {
+			d, ok := tb.take(tb.startTime.Add(req.time), req.count, infinityDuration)
+			c.Assert(ok, gc.Equals, true)
+			if d != req.expectWait {
+				c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expectWait)
+			}
+		}
+	}
+}
+
+func (rateLimitSuite) TestTakeMaxDuration(c *gc.C) {
+	for i, test := range takeTests {
+		tb := NewBucket(test.fillInterval, test.capacity)
+		for j, req := range test.reqs {
+			if req.expectWait > 0 {
+				d, ok := tb.take(tb.startTime.Add(req.time), req.count, req.expectWait-1)
+				c.Assert(ok, gc.Equals, false)
+				c.Assert(d, gc.Equals, time.Duration(0))
+			}
+			d, ok := tb.take(tb.startTime.Add(req.time), req.count, req.expectWait)
+			c.Assert(ok, gc.Equals, true)
+			if d != req.expectWait {
+				c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expectWait)
+			}
+		}
+	}
+}
+
+type takeAvailableReq struct {
+	time   time.Duration
+	count  int64
+	expect int64
+}
+
+var takeAvailableTests = []struct {
+	about        string
+	fillInterval time.Duration
+	capacity     int64
+	reqs         []takeAvailableReq
+}{{
+	about:        "serial requests",
+	fillInterval: 250 * time.Millisecond,
+	capacity:     10,
+	reqs: []takeAvailableReq{{
+		time:   0,
+		count:  0,
+		expect: 0,
+	}, {
+		time:   0,
+		count:  10,
+		expect: 10,
+	}, {
+		time:   0,
+		count:  1,
+		expect: 0,
+	}, {
+		time:   250 * time.Millisecond,
+		count:  1,
+		expect: 1,
+	}},
+}, {
+	about:        "concurrent requests",
+	fillInterval: 250 * time.Millisecond,
+	capacity:     10,
+	reqs: []takeAvailableReq{{
+		time:   0,
+		count:  5,
+		expect: 5,
+	}, {
+		time:   0,
+		count:  2,
+		expect: 2,
+	}, {
+		time:   0,
+		count:  5,
+		expect: 3,
+	}, {
+		time:   0,
+		count:  1,
+		expect: 0,
+	}},
+}, {
+	about:        "more than capacity",
+	fillInterval: 1 * time.Millisecond,
+	capacity:     10,
+	reqs: []takeAvailableReq{{
+		time:   0,
+		count:  10,
+		expect: 10,
+	}, {
+		time:   20 * time.Millisecond,
+		count:  15,
+		expect: 10,
+	}},
+}, {
+	about:        "within capacity",
+	fillInterval: 10 * time.Millisecond,
+	capacity:     5,
+	reqs: []takeAvailableReq{{
+		time:   0,
+		count:  5,
+		expect: 5,
+	}, {
+		time:   60 * time.Millisecond,
+		count:  5,
+		expect: 5,
+	}, {
+		time:   70 * time.Millisecond,
+		count:  1,
+		expect: 1,
+	}},
+}}
+
+func (rateLimitSuite) TestTakeAvailable(c *gc.C) {
+	for i, test := range takeAvailableTests {
+		tb := NewBucket(test.fillInterval, test.capacity)
+		for j, req := range test.reqs {
+			d := tb.takeAvailable(tb.startTime.Add(req.time), req.count)
+			if d != req.expect {
+				c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expect)
+			}
+		}
+	}
+}
+
+func (rateLimitSuite) TestPanics(c *gc.C) {
+	c.Assert(func() { NewBucket(0, 1) }, gc.PanicMatches, "token bucket fill interval is not > 0")
+	c.Assert(func() { NewBucket(-2, 1) }, gc.PanicMatches, "token bucket fill interval is not > 0")
+	c.Assert(func() { NewBucket(1, 0) }, gc.PanicMatches, "token bucket capacity is not > 0")
+	c.Assert(func() { NewBucket(1, -2) }, gc.PanicMatches, "token bucket capacity is not > 0")
+}
+
+func isCloseTo(x, y, tolerance float64) bool {
+	return abs(x-y)/y < tolerance
+}
+
+func (rateLimitSuite) TestRate(c *gc.C) {
+	tb := NewBucket(1, 1)
+	if !isCloseTo(tb.Rate(), 1e9, 0.00001) {
+		c.Fatalf("got %v want 1e9", tb.Rate())
+	}
+	tb = NewBucket(2*time.Second, 1)
+	if !isCloseTo(tb.Rate(), 0.5, 0.00001) {
+		c.Fatalf("got %v want 0.5", tb.Rate())
+	}
+	tb = newBucketWithQuantum(100*time.Millisecond, 1, 5)
+	if !isCloseTo(tb.Rate(), 50, 0.00001) {
+		c.Fatalf("got %v want 50", tb.Rate())
+	}
+}
+
+func checkRate(c *gc.C, rate float64) {
+	tb := NewBucketWithRate(rate, 1<<62)
+	if !isCloseTo(tb.Rate(), rate, rateMargin) {
+		c.Fatalf("got %g want %v", tb.Rate(), rate)
+	}
+	d, ok := tb.take(tb.startTime, 1<<62, infinityDuration)
+	c.Assert(ok, gc.Equals, true)
+	c.Assert(d, gc.Equals, time.Duration(0))
+
+	// Check that the actual rate is as expected by
+	// asking for a not-quite multiple of the bucket's
+	// quantum and checking that the wait time
+	// correct.
+	d, ok = tb.take(tb.startTime, tb.quantum*2-tb.quantum/2, infinityDuration)
+	c.Assert(ok, gc.Equals, true)
+	expectTime := 1e9 * float64(tb.quantum) * 2 / rate
+	if !isCloseTo(float64(d), expectTime, rateMargin) {
+		c.Fatalf("rate %g: got %g want %v", rate, float64(d), expectTime)
+	}
+}
+
+func (rateLimitSuite) TestNewWithRate(c *gc.C) {
+	for rate := float64(1); rate < 1e6; rate += 7 {
+		checkRate(c, rate)
+	}
+	for _, rate := range []float64{
+		1024 * 1024 * 1024,
+		1e-5,
+		0.9e-5,
+		0.5,
+		0.9,
+		0.9e8,
+		3e12,
+		4e18,
+	} {
+		checkRate(c, rate)
+		checkRate(c, rate/3)
+		checkRate(c, rate*1.3)
+	}
+}
+
+func BenchmarkWait(b *testing.B) {
+	tb := NewBucket(1, 16*1024)
+	for i := b.N - 1; i >= 0; i-- {
+		tb.Wait(1)
+	}
+}

Godeps/_workspace/src/github.com/juju/ratelimit/reader.go

@@ -0,0 +1,51 @@
+// Copyright 2014 Canonical Ltd.
+// Licensed under the LGPLv3 with static-linking exception.
+// See LICENCE file for details.
+
+package ratelimit
+
+import "io"
+
+type reader struct {
+	r      io.Reader
+	bucket *Bucket
+}
+
+// Reader returns a reader that is rate limited by
+// the given token bucket. Each token in the bucket
+// represents one byte.
+func Reader(r io.Reader, bucket *Bucket) io.Reader {
+	return &reader{
+		r:      r,
+		bucket: bucket,
+	}
+}
+
+func (r *reader) Read(buf []byte) (int, error) {
+	n, err := r.r.Read(buf)
+	if n <= 0 {
+		return n, err
+	}
+	r.bucket.Wait(int64(n))
+	return n, err
+}
+
+type writer struct {
+	w      io.Writer
+	bucket *Bucket
+}
+
+// Writer returns a reader that is rate limited by
+// the given token bucket. Each token in the bucket
+// represents one byte.
+func Writer(w io.Writer, bucket *Bucket) io.Writer {
+	return &writer{
+		w:      w,
+		bucket: bucket,
+	}
+}
+
+func (w *writer) Write(buf []byte) (int, error) {
+	w.bucket.Wait(int64(len(buf)))
+	return w.w.Write(buf)
+}

LICENSE

@@ -1,4 +1,4 @@
-Copyright (C) 2013 Jakob Borg
+Copyright (C) 2014 Jakob Borg and other contributors (see the CONTRIBUTORS file).
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

README.md

@@ -1,18 +1,18 @@
-syncthing
+syncthing [![Build Status](https://drone.io/github.com/calmh/syncthing/status.png)](https://drone.io/github.com/calmh/syncthing/latest)
 =========
 
-This is `syncthing`, an open BitTorrent Sync alternative. It is
-currently far from ready for mass consumption, but it is a usable proof
-of concept and tech demo. The following are the project goals:
+This is the `syncthing` project. The following are the project goals:
 
- 1. Define an open, secure, language neutral protocol usable for
-    efficient synchronization of a file repository between an arbitrary
-    number of nodes. This is the [Block Exchange
-    Protocol](https://github.com/calmh/syncthing/blob/master/protocol/PROTOCOL.md)
-    (BEP).
+ 1. Define a protocol for synchronization of a file repository between a
+    number of collaborating nodes. The protocol should be well defined,
+    unambigous, easily understood, free to use, efficient, secure and
+    languange neutral. This is the [Block Exchange
+    Protocol](https://github.com/calmh/syncthing/blob/master/protocol/PROTOCOL.md).
 
  2. Provide the reference implementation to demonstrate the usability of
-    said protocol. This is the `syncthing` utility.
+    said protocol. This is the `syncthing` utility. It is the hope that
+    alternative, compatible implementations of the protocol will come to
+    exist.
 
 The two are evolving together; the protocol is not to be considered
 stable until syncthing 1.0 is released, at which point it is locked down
@@ -25,149 +25,25 @@ making sure large swarms of selfish agents behave and somehow work
 towards a common goal. Here we have a much smaller swarm of cooperative
 agents and a simpler approach will suffice.
 
-Features
---------
+Signed Releases
+---------------
 
-The following features are _currently implemented and working_:
+As of v0.7.0 and onwards, git tags and release binaries are GPG signed with
+the key BCE524C7 (http://nym.se/gpg.txt). The signature is included in the
+normal release bundle as `syncthing.asc` or `syncthing.exe.asc`.
 
- * The formation of a cluster of nodes, certificate authenticated and
-   communicating over TLS over TCP.
+Documentation
+=============
 
- * Synchronization of a single directory among the cluster nodes.
-
- * Change detection by periodic scanning of the local repository.
-
- * Static configuration of cluster nodes.
-
- * Automatic discovery of cluster nodes. See [discover.go][discover.go]
-   for the protocol specification. Discovery on the LAN is performed by
-   broadcasts, Internet wide discovery is performed with the assistance
-   of a global server.
-
- * Handling of deleted files. Deletes can be propagated or ignored per
-   client.
-
- * Synchronizing multiple unrelated directory trees by following
-   symlinks directly below the repository level.
-
-The following features are _not yet implemented but planned_:
-
- * Change detection by listening to file system notifications instead of
-   periodic scanning.
-
- * HTTP GUI.
-
-The following features are _not implemented but may be implemented_ in
-the future:
-
- * Syncing multiple directories from the same syncthing instance.
-
- * Automatic NAT handling via UPNP.
-
- * Conflict resolution. Currently whichever file has the newest
-   modification time "wins". The correct behavior in the face of
-   conflicts is open for discussion.
-
-[discover.go]: (https://github.com/calmh/syncthing/blob/master/discover/discover.go
-
-Security
---------
-
-Security is one of the primary project goals. This means that it should
-not be possible for an attacker to join a cluster uninvited, and it
-should not be possible to extract private information from intercepted
-traffic. Currently this is implemented as follows.
-
-All traffic is protected by TLS. To prevent uninvited nodes from joining
-a cluster, the certificate fingerprint of each node is compared to a
-preset list of acceptable nodes at connection establishment. The
-fingerprint is computed as the SHA-1 hash of the certificate and
-displayed in BASE32 encoding to form a compact yet convenient string.
-Currently SHA-1 is deemed secure against preimage attacks.
-
-Installing
-==========
-
-Download the appropriate precompiled binary from the
-[releases](https://github.com/calmh/syncthing/releases) page. Untar and
-put the `syncthing` binary somewhere convenient in your `$PATH`.
-
-If you are a developer and have Go 1.2 installed you can also install
-the latest version from source:
-
-`go get github.com/calmh/syncthing`
-
-Usage
-=====
-
-Check out the options:
-
-```
-$ syncthing --help
-Usage:
-  syncthing [options]
-
-...
-```
-
-Run syncthing to let it create it's config directory and certificate:
-
-```
-$ syncthing
-11:34:13 main.go:85: INFO: Version v0.1-40-gbb0fd87
-11:34:13 tls.go:61: OK: wrote cert.pem
-11:34:13 tls.go:67: OK: wrote key.pem
-11:34:13 main.go:66: INFO: My ID: NCTBZAAHXR6ZZP3D7SL3DLYFFQERMW4Q
-11:34:13 main.go:90: FATAL: No config file
-```
-
-Take note of the "My ID: ..." line. Perform the same operation on
-another computer to create another node. Take note of that ID as well,
-and create a config file `~/.syncthing/syncthing.ini` looking something
-like this:
-
-```
-[repository]
-dir = /Users/jb/Synced
-
-[nodes]
-NCTBZAAHXR6ZZP3D7SL3DLYFFQERMW4Q = 172.16.32.1:22000 192.23.34.56:22000
-CUGAE43Y5N64CRJU26YFH6MTWPSBLSUL = dynamic
-```
-
-This assumes that the first node is reachable on either of the two
-addresses listed (perhaps one internal and one port-forwarded external)
-and that the other node is not normally reachable from the outside. Save
-this config file, identically, to both nodes.
-
-If the nodes are running on the same network, or reachable on port 22000
-from the outside world, you can set all addresses to "dynamic" and they
-will find each other using automatic discovery. (This discovery,
-including port numbers, can be tweaked or disabled using command line
-options.)
-
-Start syncthing on both nodes. For the cautious, one side can be set to
-be read only.
-
-```
-$ syncthing --ro
-13:30:55 main.go:85: INFO: Version v0.1-40-gbb0fd87
-13:30:55 main.go:102: INFO: My ID: NCTBZAAHXR6ZZP3D7SL3DLYFFQERMW4Q
-13:30:55 main.go:149: INFO: Initial repository scan in progress
-13:30:59 main.go:153: INFO: Listening for incoming connections
-13:30:59 main.go:157: INFO: Attempting to connect to other nodes
-13:30:59 main.go:247: INFO: Starting local discovery
-13:30:59 main.go:165: OK: Ready to synchronize
-13:31:04 discover.go:113: INFO: Discovered node CUGAE43Y5N64CRJU26YFH6MTWPSBLSUL at 172.16.32.24:22000
-13:31:14 main.go:296: OK: Connected to node CUGAE43Y5N64CRJU26YFH6MTWPSBLSUL
-13:31:19 main.go:345: INFO: Transferred 139 KiB in (14 KiB/s), 139 KiB out (14 KiB/s)
-...
-```
-You should see the synchronization start and then finish a short while
-later. Add nodes to taste.
+The syncthing documentation is kept on the
+[GitHub Wiki](https://github.com/calmh/syncthing/wiki).
 
 License
 =======
 
-MIT
+All documentation and protocol specifications are licensed
+under the [Creative Commons Attribution 4.0 International
+License](http://creativecommons.org/licenses/by/4.0/).
 
+All code is licensed under the [MIT
+License](https://github.com/calmh/syncthing/blob/master/LICENSE).

auto/doc.go

@@ -0,0 +1,2 @@
+// Package auto contains auto generated files for web assets.
+package auto

blocks.go

@@ -1,67 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"io"
-)
-
-type BlockList []Block
-
-type Block struct {
-	Offset uint64
-	Length uint32
-	Hash   []byte
-}
-
-// Blocks returns the blockwise hash of the reader.
-func Blocks(r io.Reader, blocksize int) (BlockList, error) {
-	var blocks BlockList
-	var offset uint64
-	for {
-		lr := &io.LimitedReader{r, int64(blocksize)}
-		hf := sha256.New()
-		n, err := io.Copy(hf, lr)
-		if err != nil {
-			return nil, err
-		}
-
-		if n == 0 {
-			break
-		}
-
-		b := Block{
-			Offset: offset,
-			Length: uint32(n),
-			Hash:   hf.Sum(nil),
-		}
-		blocks = append(blocks, b)
-		offset += uint64(n)
-	}
-
-	return blocks, nil
-}
-
-// To returns the list of blocks necessary to transform src into dst.
-// Both block lists must have been created with the same block size.
-func (src BlockList) To(tgt BlockList) (have, need BlockList) {
-	if len(tgt) == 0 && len(src) != 0 {
-		return nil, nil
-	}
-
-	if len(tgt) != 0 && len(src) == 0 {
-		// Copy the entire file
-		return nil, tgt
-	}
-
-	for i := range tgt {
-		if i >= len(src) || bytes.Compare(tgt[i].Hash, src[i].Hash) != 0 {
-			// Copy differing block
-			need = append(need, tgt[i])
-		} else {
-			have = append(have, tgt[i])
-		}
-	}
-
-	return have, need
-}

buffers/buffers.go

@@ -1,3 +1,4 @@
+// Package buffers manages a set of reusable byte buffers.
 package buffers
 
 const (

build.sh

@@ -1,25 +1,132 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
-version=$(git describe --always)
+export COPYFILE_DISABLE=true
 
-go test ./... || exit 1
+distFiles=(README.md LICENSE) # apart from the binary itself
+version=$(git describe --always --dirty)
 
-rm -rf build
-mkdir -p build || exit 1
+build() {
+	if command -v godep >/dev/null ; then
+		godep=godep
+	else
+		echo "Warning: no godep, using \"go get\" instead."
+		echo "Try \"go get github.com/tools/godep\"."
+		go get -d ./cmd/syncthing
+		godep=
+	fi
+	${godep} go build $* -ldflags "-w -X main.Version $version" ./cmd/syncthing
+	${godep} go build -ldflags "-w -X main.Version $version" ./cmd/stcli
+}
 
-for goos in darwin linux freebsd ; do
-	for goarch in amd64 386 ; do
-		echo "$goos-$goarch"
-		export GOOS="$goos"
-		export GOARCH="$goarch"
-		export name="syncthing-$goos-$goarch"
-		go build -ldflags "-X main.Version $version" \
-		&& mkdir -p "$name" \
-		&& cp syncthing "build/$name" \
-		&& cp README.md LICENSE "$name" \
-		&& mv syncthing "$name" \
-		&& tar zcf "$name.tar.gz" "$name" \
-		&& rm -r  "$name" \
-		&& mv "$name.tar.gz" build
-	done
-done
+prepare() {
+	go run cmd/assets/assets.go gui > auto/gui.files.go
+}
+
+test() {
+	go test -cpu=1,2,4 ./...
+}
+
+sign() {
+	if git describe --exact-match 2>/dev/null >/dev/null ; then
+		# HEAD is a tag
+		id=BCE524C7
+		if gpg --list-keys "$id" >/dev/null 2>&1 ; then
+			gpg -ab -u "$id" "$1"
+		fi
+	fi
+}
+
+tarDist() {
+	name="$1"
+	rm -rf "$name"
+	mkdir -p "$name"
+	cp syncthing "${distFiles[@]}" "$name"
+	sign "$name/syncthing"
+	tar zcvf "$name.tar.gz" "$name"
+	rm -rf "$name"
+}
+
+zipDist() {
+	name="$1"
+	rm -rf "$name"
+	mkdir -p "$name"
+	cp syncthing.exe "${distFiles[@]}" "$name"
+	sign "$name/syncthing.exe"
+	zip -r "$name.zip" "$name"
+	rm -rf "$name"
+}
+
+case "$1" in
+	"")
+		build
+		;;
+
+	race)
+		build -race
+		;;
+
+	test)
+		test
+		;;
+
+	tar)
+		rm -f *.tar.gz *.zip
+		prepare
+		test || exit 1
+		build
+
+		eval $(go env)
+		name="syncthing-$GOOS-$GOARCH-$version"
+
+		tarDist "$name"
+		;;
+
+	all)
+		rm -f *.tar.gz *.zip
+		prepare
+		test || exit 1
+
+		for os in darwin-amd64 linux-386 linux-amd64 freebsd-amd64 windows-amd64 ; do
+			export GOOS=${os%-*}
+			export GOARCH=${os#*-}
+
+			build
+
+			name="syncthing-$os-$version"
+			case $GOOS in
+				windows)
+					zipDist "$name"
+					rm -f syncthing.exe
+					;;
+				*)
+					tarDist "$name"
+					rm -f syncthing
+					;;
+			esac
+		done
+
+		export GOOS=linux
+		export GOARCH=arm
+
+		export GOARM=7
+		build
+		tarDist "syncthing-linux-armv7-$version"
+
+		export GOARM=6
+		build
+		tarDist "syncthing-linux-armv6-$version"
+
+		;;
+
+	upload)
+		tag=$(git describe)
+		shopt -s nullglob
+		for f in *.tar.gz *.zip *.asc ; do
+			relup calmh/syncthing "$tag" "$f"
+		done
+		;;
+
+	*)
+		echo "Unknown build parameter $1"
+		;;
+esac

cid/cid.go

@@ -0,0 +1,78 @@
+// Package cid provides a manager for mappings between node ID:s and connection ID:s.
+package cid
+
+import "sync"
+
+type Map struct {
+	sync.Mutex
+	toCid  map[string]uint
+	toName []string
+}
+
+var (
+	LocalName      = "<local>"
+	LocalID   uint = 0
+)
+
+func NewMap() *Map {
+	return &Map{
+		toCid:  map[string]uint{"<local>": 0},
+		toName: []string{"<local>"},
+	}
+}
+
+func (m *Map) Get(name string) uint {
+	m.Lock()
+	defer m.Unlock()
+
+	cid, ok := m.toCid[name]
+	if ok {
+		return cid
+	}
+
+	// Find a free slot to get a new ID
+	for i, n := range m.toName {
+		if n == "" {
+			m.toName[i] = name
+			m.toCid[name] = uint(i)
+			return uint(i)
+		}
+	}
+
+	// Add it to the end since we didn't find a free slot
+	m.toName = append(m.toName, name)
+	cid = uint(len(m.toName) - 1)
+	m.toCid[name] = cid
+	return cid
+}
+
+func (m *Map) Name(cid uint) string {
+	m.Lock()
+	defer m.Unlock()
+
+	return m.toName[cid]
+}
+
+func (m *Map) Names() []string {
+	m.Lock()
+
+	var names []string
+	for _, name := range m.toName {
+		if name != "" {
+			names = append(names, name)
+		}
+	}
+
+	m.Unlock()
+	return names
+}
+
+func (m *Map) Clear(name string) {
+	m.Lock()
+	cid, ok := m.toCid[name]
+	if ok {
+		m.toName[cid] = ""
+		delete(m.toCid, name)
+	}
+	m.Unlock()
+}

cid/cid_test.go

@@ -0,0 +1,27 @@
+package cid
+
+import "testing"
+
+func TestGet(t *testing.T) {
+	m := NewMap()
+
+	if i := m.Get("foo"); i != 1 {
+		t.Errorf("Unexpected id %d != 1", i)
+	}
+	if i := m.Get("bar"); i != 2 {
+		t.Errorf("Unexpected id %d != 2", i)
+	}
+	if i := m.Get("foo"); i != 1 {
+		t.Errorf("Unexpected id %d != 1", i)
+	}
+	if i := m.Get("bar"); i != 2 {
+		t.Errorf("Unexpected id %d != 2", i)
+	}
+
+	if LocalID != 0 {
+		t.Error("LocalID should be 0")
+	}
+	if i := m.Get(LocalName); i != LocalID {
+		t.Errorf("Unexpected id %d != %c", i, LocalID)
+	}
+}

cmd/.gitignore

@@ -0,0 +1,2 @@
+!syncthing
+!stcli

cmd/assets/assets.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"flag"
+	"fmt"
+	"go/format"
+	"io"
+	"os"
+	"path/filepath"
+	"text/template"
+)
+
+var tpl = template.Must(template.New("assets").Parse(`package auto
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/hex"
+	"io/ioutil"
+)
+
+var Assets = make(map[string][]byte)
+
+func init() {
+	var bs []byte
+	var gr *gzip.Reader
+{{range $asset := .assets}}
+	bs, _ = hex.DecodeString("{{$asset.HexData}}")
+	gr, _ = gzip.NewReader(bytes.NewBuffer(bs))
+	bs, _ = ioutil.ReadAll(gr)
+	Assets["{{$asset.Name}}"] = bs
+{{end}}
+}
+`))
+
+type asset struct {
+	Name    string
+	HexData string
+}
+
+var assets []asset
+
+func walkerFor(basePath string) filepath.WalkFunc {
+	return func(name string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.Mode().IsRegular() {
+			fd, err := os.Open(name)
+			if err != nil {
+				return err
+			}
+
+			var buf bytes.Buffer
+			gw := gzip.NewWriter(&buf)
+			io.Copy(gw, fd)
+			fd.Close()
+			gw.Flush()
+			gw.Close()
+
+			name, _ = filepath.Rel(basePath, name)
+			assets = append(assets, asset{
+				Name:    filepath.ToSlash(name),
+				HexData: fmt.Sprintf("%x", buf.Bytes()),
+			})
+		}
+
+		return nil
+	}
+}
+
+func main() {
+	flag.Parse()
+
+	filepath.Walk(flag.Arg(0), walkerFor(flag.Arg(0)))
+	var buf bytes.Buffer
+	tpl.Execute(&buf, map[string][]asset{"assets": assets})
+	bs, err := format.Source(buf.Bytes())
+	if err != nil {
+		panic(err)
+	}
+	os.Stdout.Write(bs)
+}

cmd/stcli/logger.go

@@ -6,21 +6,21 @@ import (
 	"os"
 )
 
-var debugEnabled = true
-var logger = log.New(os.Stderr, "", log.Lshortfile|log.Ltime)
+var logger *log.Logger
+
+func init() {
+	log.SetOutput(os.Stderr)
+	logger = log.New(os.Stderr, "", log.Flags())
+}
 
 func debugln(vals ...interface{}) {
-	if debugEnabled {
-		s := fmt.Sprintln(vals...)
-		logger.Output(2, "DEBUG: "+s)
-	}
+	s := fmt.Sprintln(vals...)
+	logger.Output(2, "DEBUG: "+s)
 }
 
 func debugf(format string, vals ...interface{}) {
-	if debugEnabled {
-		s := fmt.Sprintf(format, vals...)
-		logger.Output(2, "DEBUG: "+s)
-	}
+	s := fmt.Sprintf(format, vals...)
+	logger.Output(2, "DEBUG: "+s)
 }
 
 func infoln(vals ...interface{}) {

cmd/stcli/main.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"crypto/tls"
+	"flag"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/calmh/syncthing/protocol"
+)
+
+var (
+	exit    bool
+	cmd     string
+	confDir string
+	target  string
+	get     string
+	pc      protocol.Connection
+)
+
+func main() {
+	log.SetFlags(0)
+	log.SetOutput(os.Stdout)
+
+	flag.StringVar(&cmd, "cmd", "idx", "Command")
+	flag.StringVar(&confDir, "home", ".", "Certificates directory")
+	flag.StringVar(&target, "target", "127.0.0.1:22000", "Target node")
+	flag.StringVar(&get, "get", "", "Get file")
+	flag.BoolVar(&exit, "exit", false, "Exit after command")
+	flag.Parse()
+
+	connect(target)
+
+	select {}
+}
+
+func connect(target string) {
+	cert, err := loadCert(confDir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	myID := string(certID(cert.Certificate[0]))
+
+	tlsCfg := &tls.Config{
+		Certificates:           []tls.Certificate{cert},
+		NextProtos:             []string{"bep/1.0"},
+		ServerName:             myID,
+		ClientAuth:             tls.RequestClientCert,
+		SessionTicketsDisabled: true,
+		InsecureSkipVerify:     true,
+		MinVersion:             tls.VersionTLS12,
+	}
+
+	conn, err := tls.Dial("tcp", target, tlsCfg)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	remoteID := certID(conn.ConnectionState().PeerCertificates[0].Raw)
+
+	pc = protocol.NewConnection(remoteID, conn, conn, Model{}, nil)
+
+	select {}
+}
+
+type Model struct {
+}
+
+func prtIndex(files []protocol.FileInfo) {
+	for _, f := range files {
+		log.Printf("%q (v:%d mod:%d flags:0%o nblocks:%d)", f.Name, f.Version, f.Modified, f.Flags, len(f.Blocks))
+		for _, b := range f.Blocks {
+			log.Printf("    %6d %x", b.Size, b.Hash)
+		}
+	}
+}
+
+func (m Model) Index(nodeID string, repo string, files []protocol.FileInfo) {
+	log.Printf("Received index for repo %q", repo)
+	if cmd == "idx" {
+		prtIndex(files)
+		if get != "" {
+			for _, f := range files {
+				if f.Name == get {
+					go getFile(f)
+					break
+				}
+			}
+		} else if exit {
+			os.Exit(0)
+		}
+	}
+}
+
+func getFile(f protocol.FileInfo) {
+	fn := filepath.Base(f.Name)
+	fd, err := os.Create(fn)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var offset int64
+	for _, b := range f.Blocks {
+		log.Printf("Request %q %d - %d", f.Name, offset, offset+int64(b.Size))
+		bs, err := pc.Request("default", f.Name, offset, int(b.Size))
+		log.Printf(" - got %d bytes", len(bs))
+		if err != nil {
+			log.Fatal(err)
+		}
+		offset += int64(b.Size)
+		fd.Write(bs)
+	}
+
+	fd.Close()
+}
+
+func (m Model) IndexUpdate(nodeID string, repo string, files []protocol.FileInfo) {
+	log.Printf("Received index update for repo %q", repo)
+	if cmd == "idx" {
+		prtIndex(files)
+		if exit {
+			os.Exit(0)
+		}
+	}
+}
+
+func (m Model) Request(nodeID, repo string, name string, offset int64, size int) ([]byte, error) {
+	log.Println("Received request")
+	return nil, io.EOF
+}
+
+func (m Model) Close(nodeID string, err error) {
+	log.Println("Received close")
+}

cmd/stcli/tls.go

@@ -3,7 +3,7 @@ package main
 import (
 	"crypto/rand"
 	"crypto/rsa"
-	"crypto/sha1"
+	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
@@ -11,27 +11,30 @@ import (
 	"encoding/pem"
 	"math/big"
 	"os"
-	"path"
+	"path/filepath"
+	"strings"
 	"time"
 )
 
 const (
-	tlsRSABits = 2048
+	tlsRSABits = 3072
 	tlsName    = "syncthing"
 )
 
 func loadCert(dir string) (tls.Certificate, error) {
-	return tls.LoadX509KeyPair(path.Join(dir, "cert.pem"), path.Join(dir, "key.pem"))
+	return tls.LoadX509KeyPair(filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem"))
 }
 
-func certId(bs []byte) string {
-	hf := sha1.New()
+func certID(bs []byte) string {
+	hf := sha256.New()
 	hf.Write(bs)
 	id := hf.Sum(nil)
-	return base32.StdEncoding.EncodeToString(id)
+	return strings.Trim(base32.StdEncoding.EncodeToString(id), "=")
 }
 
 func newCertificate(dir string) {
+	infoln("Generating RSA certificate and key...")
+
 	priv, err := rsa.GenerateKey(rand.Reader, tlsRSABits)
 	fatalErr(err)
 
@@ -47,22 +50,22 @@ func newCertificate(dir string) {
 		NotAfter:  notAfter,
 
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
 		BasicConstraintsValid: true,
 	}
 
 	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
 	fatalErr(err)
 
-	certOut, err := os.Create(path.Join(dir, "cert.pem"))
+	certOut, err := os.Create(filepath.Join(dir, "cert.pem"))
 	fatalErr(err)
 	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
 	certOut.Close()
-	okln("wrote cert.pem")
+	okln("Created RSA certificate file")
 
-	keyOut, err := os.OpenFile(path.Join(dir, "key.pem"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	keyOut, err := os.OpenFile(filepath.Join(dir, "key.pem"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	fatalErr(err)
 	pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
 	keyOut.Close()
-	okln("wrote key.pem")
+	okln("Created RSA key file")
 }

cmd/syncthing/.gitignore

@@ -0,0 +1 @@
+*.idx.gz

cmd/syncthing/blockqueue.go

@@ -0,0 +1,94 @@
+package main
+
+import "github.com/calmh/syncthing/scanner"
+
+type bqAdd struct {
+	file scanner.File
+	have []scanner.Block
+	need []scanner.Block
+}
+
+type bqBlock struct {
+	file  scanner.File
+	block scanner.Block   // get this block from the network
+	copy  []scanner.Block // copy these blocks from the old version of the file
+	last  bool
+}
+
+type blockQueue struct {
+	inbox  chan bqAdd
+	outbox chan bqBlock
+
+	queued []bqBlock
+}
+
+func newBlockQueue() *blockQueue {
+	q := &blockQueue{
+		inbox:  make(chan bqAdd),
+		outbox: make(chan bqBlock),
+	}
+	go q.run()
+	return q
+}
+
+func (q *blockQueue) addBlock(a bqAdd) {
+	// If we already have it queued, return
+	for _, b := range q.queued {
+		if b.file.Name == a.file.Name {
+			return
+		}
+	}
+	if len(a.have) > 0 {
+		// First queue a copy operation
+		q.queued = append(q.queued, bqBlock{
+			file: a.file,
+			copy: a.have,
+		})
+	}
+	// Queue the needed blocks individually
+	l := len(a.need)
+	for i, b := range a.need {
+		q.queued = append(q.queued, bqBlock{
+			file:  a.file,
+			block: b,
+			last:  i == l-1,
+		})
+	}
+
+	if l == 0 {
+		// If we didn't have anything to fetch, queue an empty block with the "last" flag set to close the file.
+		q.queued = append(q.queued, bqBlock{
+			file: a.file,
+			last: true,
+		})
+	}
+}
+
+func (q *blockQueue) run() {
+	for {
+		if len(q.queued) == 0 {
+			q.addBlock(<-q.inbox)
+		} else {
+			next := q.queued[0]
+			select {
+			case a := <-q.inbox:
+				q.addBlock(a)
+			case q.outbox <- next:
+				q.queued = q.queued[1:]
+			}
+		}
+	}
+}
+
+func (q *blockQueue) put(a bqAdd) {
+	q.inbox <- a
+}
+
+func (q *blockQueue) get() bqBlock {
+	return <-q.outbox
+}
+
+func (q *blockQueue) empty() bool {
+	// There is a race condition here. We're only mostly sure the queue is empty if the expression below is true.
+	return len(q.queued) == 0 && len(q.inbox) == 0 && len(q.outbox) == 0
+}

cmd/syncthing/config.go

@@ -0,0 +1,243 @@
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+type Configuration struct {
+	Version      int                       `xml:"version,attr" default:"1"`
+	Repositories []RepositoryConfiguration `xml:"repository"`
+	Options      OptionsConfiguration      `xml:"options"`
+	XMLName      xml.Name                  `xml:"configuration" json:"-"`
+}
+
+type RepositoryConfiguration struct {
+	ID        string              `xml:"id,attr"`
+	Directory string              `xml:"directory,attr"`
+	Nodes     []NodeConfiguration `xml:"node"`
+}
+
+type NodeConfiguration struct {
+	NodeID    string   `xml:"id,attr"`
+	Name      string   `xml:"name,attr"`
+	Addresses []string `xml:"address"`
+}
+
+type OptionsConfiguration struct {
+	ListenAddress      []string `xml:"listenAddress" default:":22000" ini:"listen-address"`
+	ReadOnly           bool     `xml:"readOnly" ini:"read-only"`
+	FollowSymlinks     bool     `xml:"followSymlinks" default:"true" ini:"follow-symlinks"`
+	GUIEnabled         bool     `xml:"guiEnabled" default:"true" ini:"gui-enabled"`
+	GUIAddress         string   `xml:"guiAddress" default:"127.0.0.1:8080" ini:"gui-address"`
+	GlobalAnnServer    string   `xml:"globalAnnounceServer" default:"announce.syncthing.net:22025" ini:"global-announce-server"`
+	GlobalAnnEnabled   bool     `xml:"globalAnnounceEnabled" default:"true" ini:"global-announce-enabled"`
+	LocalAnnEnabled    bool     `xml:"localAnnounceEnabled" default:"true" ini:"local-announce-enabled"`
+	ParallelRequests   int      `xml:"parallelRequests" default:"16" ini:"parallel-requests"`
+	MaxSendKbps        int      `xml:"maxSendKbps" ini:"max-send-kbps"`
+	RescanIntervalS    int      `xml:"rescanIntervalS" default:"60" ini:"rescan-interval"`
+	ReconnectIntervalS int      `xml:"reconnectionIntervalS" default:"60" ini:"reconnection-interval"`
+	MaxChangeKbps      int      `xml:"maxChangeKbps" default:"1000" ini:"max-change-bw"`
+	StartBrowser       bool     `xml:"startBrowser" default:"true"`
+}
+
+func setDefaults(data interface{}) error {
+	s := reflect.ValueOf(data).Elem()
+	t := s.Type()
+
+	for i := 0; i < s.NumField(); i++ {
+		f := s.Field(i)
+		tag := t.Field(i).Tag
+
+		v := tag.Get("default")
+		if len(v) > 0 {
+			switch f.Interface().(type) {
+			case string:
+				f.SetString(v)
+
+			case int:
+				i, err := strconv.ParseInt(v, 10, 64)
+				if err != nil {
+					return err
+				}
+				f.SetInt(i)
+
+			case bool:
+				f.SetBool(v == "true")
+
+			case []string:
+				// We don't do anything with string slices here. Any default
+				// we set will be appended to by the XML decoder, so we fill
+				// those after decoding.
+
+			default:
+				panic(f.Type())
+			}
+		}
+	}
+	return nil
+}
+
+// fillNilSlices sets default value on slices that are still nil.
+func fillNilSlices(data interface{}) error {
+	s := reflect.ValueOf(data).Elem()
+	t := s.Type()
+
+	for i := 0; i < s.NumField(); i++ {
+		f := s.Field(i)
+		tag := t.Field(i).Tag
+
+		v := tag.Get("default")
+		if len(v) > 0 {
+			switch f.Interface().(type) {
+			case []string:
+				if f.IsNil() {
+					rv := reflect.MakeSlice(reflect.TypeOf([]string{}), 1, 1)
+					rv.Index(0).SetString(v)
+					f.Set(rv)
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func readConfigINI(m map[string]string, data interface{}) error {
+	s := reflect.ValueOf(data).Elem()
+	t := s.Type()
+
+	for i := 0; i < s.NumField(); i++ {
+		f := s.Field(i)
+		tag := t.Field(i).Tag
+
+		name := tag.Get("ini")
+		if len(name) == 0 {
+			name = strings.ToLower(t.Field(i).Name)
+		}
+
+		if v, ok := m[name]; ok {
+			switch f.Interface().(type) {
+			case string:
+				f.SetString(v)
+
+			case int:
+				i, err := strconv.ParseInt(v, 10, 64)
+				if err == nil {
+					f.SetInt(i)
+				}
+
+			case bool:
+				f.SetBool(v == "true")
+
+			default:
+				panic(f.Type())
+			}
+		}
+	}
+	return nil
+}
+
+func writeConfigXML(wr io.Writer, cfg Configuration) error {
+	e := xml.NewEncoder(wr)
+	e.Indent("", "    ")
+	err := e.Encode(cfg)
+	if err != nil {
+		return err
+	}
+	_, err = wr.Write([]byte("\n"))
+	return err
+}
+
+func uniqueStrings(ss []string) []string {
+	var m = make(map[string]bool, len(ss))
+	for _, s := range ss {
+		m[s] = true
+	}
+
+	var us = make([]string, 0, len(m))
+	for k := range m {
+		us = append(us, k)
+	}
+
+	return us
+}
+
+func readConfigXML(rd io.Reader) (Configuration, error) {
+	var cfg Configuration
+
+	setDefaults(&cfg)
+	setDefaults(&cfg.Options)
+
+	var err error
+	if rd != nil {
+		err = xml.NewDecoder(rd).Decode(&cfg)
+	}
+
+	fillNilSlices(&cfg.Options)
+
+	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
+
+	var seenRepos = map[string]bool{}
+	for i := range cfg.Repositories {
+		if cfg.Repositories[i].ID == "" {
+			cfg.Repositories[i].ID = "default"
+		}
+
+		id := cfg.Repositories[i].ID
+		if seenRepos[id] {
+			panic("duplicate repository ID " + id)
+		}
+		seenRepos[id] = true
+	}
+
+	return cfg, err
+}
+
+type NodeConfigurationList []NodeConfiguration
+
+func (l NodeConfigurationList) Less(a, b int) bool {
+	return l[a].NodeID < l[b].NodeID
+}
+func (l NodeConfigurationList) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+func (l NodeConfigurationList) Len() int {
+	return len(l)
+}
+
+func clusterHash(nodes []NodeConfiguration) string {
+	sort.Sort(NodeConfigurationList(nodes))
+	h := sha256.New()
+	for _, n := range nodes {
+		h.Write([]byte(n.NodeID))
+	}
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func cleanNodeList(nodes []NodeConfiguration, myID string) []NodeConfiguration {
+	var myIDExists bool
+	for _, node := range nodes {
+		if node.NodeID == myID {
+			myIDExists = true
+			break
+		}
+	}
+
+	if !myIDExists {
+		nodes = append(nodes, NodeConfiguration{
+			NodeID:    myID,
+			Addresses: []string{"dynamic"},
+			Name:      "",
+		})
+	}
+
+	sort.Sort(NodeConfigurationList(nodes))
+
+	return nodes
+}

cmd/syncthing/config_test.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"bytes"
+	"io"
+	"reflect"
+	"testing"
+)
+
+func TestDefaultValues(t *testing.T) {
+	expected := OptionsConfiguration{
+		ListenAddress:      []string{":22000"},
+		ReadOnly:           false,
+		FollowSymlinks:     true,
+		GUIEnabled:         true,
+		GUIAddress:         "127.0.0.1:8080",
+		GlobalAnnServer:    "announce.syncthing.net:22025",
+		GlobalAnnEnabled:   true,
+		LocalAnnEnabled:    true,
+		ParallelRequests:   16,
+		MaxSendKbps:        0,
+		RescanIntervalS:    60,
+		ReconnectIntervalS: 60,
+		MaxChangeKbps:      1000,
+		StartBrowser:       true,
+	}
+
+	cfg, err := readConfigXML(bytes.NewReader(nil))
+	if err != io.EOF {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(cfg.Options, expected) {
+		t.Errorf("Default config differs;\n  E: %#v\n  A: %#v", expected, cfg.Options)
+	}
+}
+
+func TestNoListenAddress(t *testing.T) {
+	data := []byte(`<configuration version="1">
+    <repository directory="~/Sync">
+        <node id="..." name="...">
+            <address>dynamic</address>
+        </node>
+    </repository>
+    <options>
+        <listenAddress></listenAddress>
+    </options>
+</configuration>
+`)
+
+	cfg, err := readConfigXML(bytes.NewReader(data))
+	if err != nil {
+		t.Error(err)
+	}
+
+	expected := []string{""}
+	if !reflect.DeepEqual(cfg.Options.ListenAddress, expected) {
+		t.Errorf("Unexpected ListenAddress %#v", cfg.Options.ListenAddress)
+	}
+}
+
+func TestOverriddenValues(t *testing.T) {
+	data := []byte(`<configuration version="1">
+    <repository directory="~/Sync">
+        <node id="..." name="...">
+            <address>dynamic</address>
+        </node>
+    </repository>
+    <options>
+       <listenAddress>:23000</listenAddress>
+        <readOnly>true</readOnly>
+        <allowDelete>false</allowDelete>
+        <followSymlinks>false</followSymlinks>
+        <guiEnabled>false</guiEnabled>
+        <guiAddress>125.2.2.2:8080</guiAddress>
+        <globalAnnounceServer>syncthing.nym.se:22025</globalAnnounceServer>
+        <globalAnnounceEnabled>false</globalAnnounceEnabled>
+        <localAnnounceEnabled>false</localAnnounceEnabled>
+        <parallelRequests>32</parallelRequests>
+        <maxSendKbps>1234</maxSendKbps>
+        <rescanIntervalS>600</rescanIntervalS>
+        <reconnectionIntervalS>6000</reconnectionIntervalS>
+        <maxChangeKbps>2345</maxChangeKbps>
+        <startBrowser>false</startBrowser>
+    </options>
+</configuration>
+`)
+
+	expected := OptionsConfiguration{
+		ListenAddress:      []string{":23000"},
+		ReadOnly:           true,
+		FollowSymlinks:     false,
+		GUIEnabled:         false,
+		GUIAddress:         "125.2.2.2:8080",
+		GlobalAnnServer:    "syncthing.nym.se:22025",
+		GlobalAnnEnabled:   false,
+		LocalAnnEnabled:    false,
+		ParallelRequests:   32,
+		MaxSendKbps:        1234,
+		RescanIntervalS:    600,
+		ReconnectIntervalS: 6000,
+		MaxChangeKbps:      2345,
+		StartBrowser:       false,
+	}
+
+	cfg, err := readConfigXML(bytes.NewReader(data))
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(cfg.Options, expected) {
+		t.Errorf("Overridden config differs;\n  E: %#v\n  A: %#v", expected, cfg.Options)
+	}
+}

cmd/syncthing/debug.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"log"
+	"os"
+	"strings"
+)
+
+var (
+	dlog      = log.New(os.Stderr, "main: ", log.Lmicroseconds|log.Lshortfile)
+	debugNet  = strings.Contains(os.Getenv("STTRACE"), "net")
+	debugIdx  = strings.Contains(os.Getenv("STTRACE"), "idx")
+	debugNeed = strings.Contains(os.Getenv("STTRACE"), "need")
+	debugPull = strings.Contains(os.Getenv("STTRACE"), "pull")
+)

cmd/syncthing/gui.go

@@ -0,0 +1,190 @@
+package main
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/calmh/syncthing/scanner"
+	"github.com/codegangsta/martini"
+)
+
+type guiError struct {
+	Time  time.Time
+	Error string
+}
+
+var (
+	configInSync = true
+	guiErrors    = []guiError{}
+	guiErrorsMut sync.Mutex
+)
+
+func startGUI(addr string, m *Model) {
+	router := martini.NewRouter()
+	router.Get("/", getRoot)
+	router.Get("/rest/version", restGetVersion)
+	router.Get("/rest/model", restGetModel)
+	router.Get("/rest/connections", restGetConnections)
+	router.Get("/rest/config", restGetConfig)
+	router.Get("/rest/config/sync", restGetConfigInSync)
+	router.Get("/rest/need", restGetNeed)
+	router.Get("/rest/system", restGetSystem)
+	router.Get("/rest/errors", restGetErrors)
+
+	router.Post("/rest/config", restPostConfig)
+	router.Post("/rest/restart", restPostRestart)
+	router.Post("/rest/reset", restPostReset)
+	router.Post("/rest/error", restPostError)
+
+	go func() {
+		mr := martini.New()
+		mr.Use(embeddedStatic())
+		mr.Use(martini.Recovery())
+		mr.Use(restMiddleware)
+		mr.Action(router.Handle)
+		mr.Map(m)
+		err := http.ListenAndServe(addr, mr)
+		if err != nil {
+			warnln("GUI not possible:", err)
+		}
+	}()
+}
+
+func getRoot(w http.ResponseWriter, r *http.Request) {
+	http.Redirect(w, r, "/index.html", 302)
+}
+
+func restMiddleware(w http.ResponseWriter, r *http.Request) {
+	if len(r.URL.Path) >= 6 && r.URL.Path[:6] == "/rest/" {
+		w.Header().Set("Cache-Control", "no-cache")
+	}
+}
+
+func restGetVersion() string {
+	return Version
+}
+
+func restGetModel(m *Model, w http.ResponseWriter) {
+	var res = make(map[string]interface{})
+
+	globalFiles, globalDeleted, globalBytes := m.GlobalSize()
+	res["globalFiles"], res["globalDeleted"], res["globalBytes"] = globalFiles, globalDeleted, globalBytes
+
+	localFiles, localDeleted, localBytes := m.LocalSize()
+	res["localFiles"], res["localDeleted"], res["localBytes"] = localFiles, localDeleted, localBytes
+
+	inSyncFiles, inSyncBytes := m.InSyncSize()
+	res["inSyncFiles"], res["inSyncBytes"] = inSyncFiles, inSyncBytes
+
+	files, total := m.NeedFiles()
+	res["needFiles"], res["needBytes"] = len(files), total
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(res)
+}
+
+func restGetConnections(m *Model, w http.ResponseWriter) {
+	var res = m.ConnectionStats()
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(res)
+}
+
+func restGetConfig(w http.ResponseWriter) {
+	json.NewEncoder(w).Encode(cfg)
+}
+
+func restPostConfig(req *http.Request) {
+	err := json.NewDecoder(req.Body).Decode(&cfg)
+	if err != nil {
+		log.Println(err)
+	} else {
+		saveConfig()
+		configInSync = false
+	}
+}
+
+func restGetConfigInSync(w http.ResponseWriter) {
+	json.NewEncoder(w).Encode(map[string]bool{"configInSync": configInSync})
+}
+
+func restPostRestart(req *http.Request) {
+	go restart()
+}
+
+func restPostReset(req *http.Request) {
+	resetRepositories()
+	go restart()
+}
+
+type guiFile scanner.File
+
+func (f guiFile) MarshalJSON() ([]byte, error) {
+	type t struct {
+		Name     string
+		Size     int64
+		Modified int64
+		Flags    uint32
+	}
+	return json.Marshal(t{
+		Name:     f.Name,
+		Size:     scanner.File(f).Size,
+		Modified: f.Modified,
+		Flags:    f.Flags,
+	})
+}
+
+func restGetNeed(m *Model, w http.ResponseWriter) {
+	files, _ := m.NeedFiles()
+	gfs := make([]guiFile, len(files))
+	for i, f := range files {
+		gfs[i] = guiFile(f)
+	}
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(gfs)
+}
+
+var cpuUsagePercent float64
+var cpuUsageLock sync.RWMutex
+
+func restGetSystem(w http.ResponseWriter) {
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+
+	res := make(map[string]interface{})
+	res["myID"] = myID
+	res["goroutines"] = runtime.NumGoroutine()
+	res["alloc"] = m.Alloc
+	res["sys"] = m.Sys
+	cpuUsageLock.RLock()
+	res["cpuPercent"] = cpuUsagePercent
+	cpuUsageLock.RUnlock()
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(res)
+}
+
+func restGetErrors(w http.ResponseWriter) {
+	guiErrorsMut.Lock()
+	json.NewEncoder(w).Encode(guiErrors)
+	guiErrorsMut.Unlock()
+}
+
+func restPostError(req *http.Request) {
+	bs, _ := ioutil.ReadAll(req.Body)
+	req.Body.Close()
+	showGuiError(string(bs))
+}
+
+func showGuiError(err string) {
+	guiErrorsMut.Lock()
+	guiErrors = append(guiErrors, guiError{time.Now(), err})
+	if len(guiErrors) > 5 {
+		guiErrors = guiErrors[len(guiErrors)-5:]
+	}
+	guiErrorsMut.Unlock()
+}

cmd/syncthing/gui_development.go

@@ -0,0 +1,9 @@
+//+build guidev
+
+package main
+
+import "github.com/codegangsta/martini"
+
+func embeddedStatic() interface{} {
+	return martini.Static("gui")
+}

cmd/syncthing/gui_embedded.go

@@ -0,0 +1,40 @@
+//+build !guidev
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"mime"
+	"net/http"
+	"path/filepath"
+	"time"
+
+	"github.com/calmh/syncthing/auto"
+)
+
+func embeddedStatic() interface{} {
+	var modt = time.Now().UTC().Format(http.TimeFormat)
+
+	return func(res http.ResponseWriter, req *http.Request, log *log.Logger) {
+		file := req.URL.Path
+
+		if file[0] == '/' {
+			file = file[1:]
+		}
+
+		bs, ok := auto.Assets[file]
+		if !ok {
+			return
+		}
+
+		mtype := mime.TypeByExtension(filepath.Ext(req.URL.Path))
+		if len(mtype) != 0 {
+			res.Header().Set("Content-Type", mtype)
+		}
+		res.Header().Set("Content-Size", fmt.Sprintf("%d", len(bs)))
+		res.Header().Set("Last-Modified", modt)
+
+		res.Write(bs)
+	}
+}

cmd/syncthing/gui_solaris.go

@@ -0,0 +1,92 @@
+//+build solaris
+
+package main
+
+import (
+	"encoding/binary"
+	"fmt"
+	"os"
+	"time"
+)
+
+type id_t int32
+type ulong_t uint32
+
+type timestruc_t struct {
+	Tv_sec  int64
+	Tv_nsec int64
+}
+
+func (tv timestruc_t) Nano() int64 {
+	return tv.Tv_sec*1e9 + tv.Tv_nsec
+}
+
+type prusage_t struct {
+	Pr_lwpid    id_t        /* lwp id.  0: process or defunct */
+	Pr_count    int32       /* number of contributing lwps */
+	Pr_tstamp   timestruc_t /* real time stamp, time of read() */
+	Pr_create   timestruc_t /* process/lwp creation time stamp */
+	Pr_term     timestruc_t /* process/lwp termination time stamp */
+	Pr_rtime    timestruc_t /* total lwp real (elapsed) time */
+	Pr_utime    timestruc_t /* user level CPU time */
+	Pr_stime    timestruc_t /* system call CPU time */
+	Pr_ttime    timestruc_t /* other system trap CPU time */
+	Pr_tftime   timestruc_t /* text page fault sleep time */
+	Pr_dftime   timestruc_t /* data page fault sleep time */
+	Pr_kftime   timestruc_t /* kernel page fault sleep time */
+	Pr_ltime    timestruc_t /* user lock wait sleep time */
+	Pr_slptime  timestruc_t /* all other sleep time */
+	Pr_wtime    timestruc_t /* wait-cpu (latency) time */
+	Pr_stoptime timestruc_t /* stopped time */
+	Pr_minf     ulong_t     /* minor page faults */
+	Pr_majf     ulong_t     /* major page faults */
+	Pr_nswap    ulong_t     /* swaps */
+	Pr_inblk    ulong_t     /* input blocks */
+	Pr_oublk    ulong_t     /* output blocks */
+	Pr_msnd     ulong_t     /* messages sent */
+	Pr_mrcv     ulong_t     /* messages received */
+	Pr_sigs     ulong_t     /* signals received */
+	Pr_vctx     ulong_t     /* voluntary context switches */
+	Pr_ictx     ulong_t     /* involuntary context switches */
+	Pr_sysc     ulong_t     /* system calls */
+	Pr_ioch     ulong_t     /* chars read and written */
+
+}
+
+func solarisPrusage(pid int, rusage *prusage_t) error {
+	fd, err := os.Open(fmt.Sprintf("/proc/%d/usage", pid))
+	if err != nil {
+		return err
+	}
+	err = binary.Read(fd, binary.LittleEndian, rusage)
+	fd.Close()
+	return err
+}
+
+func init() {
+	go trackCPUUsage()
+}
+
+func trackCPUUsage() {
+	var prevUsage int64
+	var prevTime = time.Now().UnixNano()
+	var rusage prusage_t
+	var pid = os.Getpid()
+	for {
+		time.Sleep(10 * time.Second)
+		err := solarisPrusage(pid, &rusage)
+		if err != nil {
+			warnln(err)
+			continue
+		}
+		curTime := time.Now().UnixNano()
+		timeDiff := curTime - prevTime
+		curUsage := rusage.Pr_utime.Nano() + rusage.Pr_stime.Nano()
+		usageDiff := curUsage - prevUsage
+		cpuUsageLock.Lock()
+		cpuUsagePercent = 100 * float64(usageDiff) / float64(timeDiff)
+		cpuUsageLock.Unlock()
+		prevTime = curTime
+		prevUsage = curUsage
+	}
+}

cmd/syncthing/gui_unix.go

@@ -0,0 +1,31 @@
+//+build !windows,!solaris
+
+package main
+
+import (
+	"syscall"
+	"time"
+)
+
+func init() {
+	go trackCPUUsage()
+}
+
+func trackCPUUsage() {
+	var prevUsage int64
+	var prevTime = time.Now().UnixNano()
+	var rusage syscall.Rusage
+	for {
+		time.Sleep(10 * time.Second)
+		syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
+		curTime := time.Now().UnixNano()
+		timeDiff := curTime - prevTime
+		curUsage := rusage.Utime.Nano() + rusage.Stime.Nano()
+		usageDiff := curUsage - prevUsage
+		cpuUsageLock.Lock()
+		cpuUsagePercent = 100 * float64(usageDiff) / float64(timeDiff)
+		cpuUsageLock.Unlock()
+		prevTime = curTime
+		prevUsage = curUsage
+	}
+}

cmd/syncthing/limitedwriter.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"io"
+
+	"github.com/juju/ratelimit"
+)
+
+type limitedWriter struct {
+	w      io.Writer
+	bucket *ratelimit.Bucket
+}
+
+func (w *limitedWriter) Write(buf []byte) (int, error) {
+	if w.bucket != nil {
+		w.bucket.Wait(int64(len(buf)))
+	}
+	return w.w.Write(buf)
+}

cmd/syncthing/logger.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+)
+
+var logger *log.Logger
+
+func init() {
+	log.SetOutput(os.Stderr)
+	logger = log.New(os.Stderr, "", log.Flags())
+}
+
+func infoln(vals ...interface{}) {
+	s := fmt.Sprintln(vals...)
+	logger.Output(2, "INFO: "+s)
+}
+
+func infof(format string, vals ...interface{}) {
+	s := fmt.Sprintf(format, vals...)
+	logger.Output(2, "INFO: "+s)
+}
+
+func okln(vals ...interface{}) {
+	s := fmt.Sprintln(vals...)
+	logger.Output(2, "OK: "+s)
+}
+
+func okf(format string, vals ...interface{}) {
+	s := fmt.Sprintf(format, vals...)
+	logger.Output(2, "OK: "+s)
+}
+
+func warnln(vals ...interface{}) {
+	s := fmt.Sprintln(vals...)
+	showGuiError(s)
+	logger.Output(2, "WARNING: "+s)
+}
+
+func warnf(format string, vals ...interface{}) {
+	s := fmt.Sprintf(format, vals...)
+	showGuiError(s)
+	logger.Output(2, "WARNING: "+s)
+}
+
+func fatalln(vals ...interface{}) {
+	s := fmt.Sprintln(vals...)
+	logger.Output(2, "FATAL: "+s)
+	os.Exit(3)
+}
+
+func fatalf(format string, vals ...interface{}) {
+	s := fmt.Sprintf(format, vals...)
+	logger.Output(2, "FATAL: "+s)
+	os.Exit(3)
+}
+
+func fatalErr(err error) {
+	if err != nil {
+		fatalf(err.Error())
+	}
+}

cmd/syncthing/main.go

@@ -0,0 +1,535 @@
+package main
+
+import (
+	"crypto/tls"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	_ "net/http/pprof"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"runtime/debug"
+	"strings"
+	"time"
+
+	"github.com/calmh/ini"
+	"github.com/calmh/syncthing/discover"
+	"github.com/calmh/syncthing/protocol"
+	"github.com/juju/ratelimit"
+)
+
+const BlockSize = 128 * 1024
+
+var cfg Configuration
+var Version = "unknown-dev"
+
+var (
+	myID       string
+	confDir    string
+	rateBucket *ratelimit.Bucket
+)
+
+const (
+	usage      = "syncthing [options]"
+	extraUsage = `The following enviroment variables are interpreted by syncthing:
+
+ STNORESTART  Do not attempt to restart when requested to, instead just exit.
+              Set this variable when running under a service manager such as
+              runit, launchd, etc.
+
+ STPROFILER   Set to a listen address such as "127.0.0.1:9090" to start the
+              profiler with HTTP access.
+
+ STTRACE      A comma separated string of facilities to trace. The valid
+              facility strings:
+              - "discover" (the node discovery package)
+              - "files"    (file set store)
+              - "idx"      (index sending and receiving)
+              - "mc"       (multicast beacon)
+              - "need"     (file need calculations)
+              - "net"      (connecting and disconnecting, network messages)
+              - "pull"     (file pull activity)
+              - "scanner"  (the file change scanner)
+              `
+)
+
+func main() {
+	var reset bool
+	var showVersion bool
+	flag.StringVar(&confDir, "home", getDefaultConfDir(), "Set configuration directory")
+	flag.BoolVar(&reset, "reset", false, "Prepare to resync from cluster")
+	flag.BoolVar(&showVersion, "version", false, "Show version")
+	flag.Usage = usageFor(flag.CommandLine, usage, extraUsage)
+	flag.Parse()
+
+	if len(os.Getenv("STRESTART")) > 0 {
+		// Give the parent process time to exit and release sockets etc.
+		time.Sleep(1 * time.Second)
+	}
+
+	if showVersion {
+		fmt.Printf("syncthing %s (%s %s-%s)\n", Version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
+		os.Exit(0)
+	}
+
+	if len(os.Getenv("GOGC")) == 0 {
+		debug.SetGCPercent(25)
+	}
+
+	if len(os.Getenv("GOMAXPROCS")) == 0 {
+		runtime.GOMAXPROCS(runtime.NumCPU())
+	}
+
+	confDir = expandTilde(confDir)
+
+	// Ensure that our home directory exists and that we have a certificate and key.
+
+	ensureDir(confDir, 0700)
+	cert, err := loadCert(confDir)
+	if err != nil {
+		newCertificate(confDir)
+		cert, err = loadCert(confDir)
+		fatalErr(err)
+	}
+
+	myID = string(certID(cert.Certificate[0]))
+	log.SetPrefix("[" + myID[0:5] + "] ")
+	logger.SetPrefix("[" + myID[0:5] + "] ")
+
+	infoln("Version", Version)
+	infoln("My ID:", myID)
+
+	// Prepare to be able to save configuration
+
+	cfgFile := filepath.Join(confDir, "config.xml")
+	go saveConfigLoop(cfgFile)
+
+	// Load the configuration file, if it exists.
+	// If it does not, create a template.
+
+	cf, err := os.Open(cfgFile)
+	if err == nil {
+		// Read config.xml
+		cfg, err = readConfigXML(cf)
+		if err != nil {
+			fatalln(err)
+		}
+		cf.Close()
+	} else {
+		// No config.xml, let's try the old syncthing.ini
+		iniFile := filepath.Join(confDir, "syncthing.ini")
+		cf, err := os.Open(iniFile)
+		if err == nil {
+			infoln("Migrating syncthing.ini to config.xml")
+			iniCfg := ini.Parse(cf)
+			cf.Close()
+			Rename(iniFile, filepath.Join(confDir, "migrated_syncthing.ini"))
+
+			cfg, _ = readConfigXML(nil)
+			cfg.Repositories = []RepositoryConfiguration{
+				{Directory: iniCfg.Get("repository", "dir")},
+			}
+			readConfigINI(iniCfg.OptionMap("settings"), &cfg.Options)
+			for name, addrs := range iniCfg.OptionMap("nodes") {
+				n := NodeConfiguration{
+					NodeID:    name,
+					Addresses: strings.Fields(addrs),
+				}
+				cfg.Repositories[0].Nodes = append(cfg.Repositories[0].Nodes, n)
+			}
+
+			saveConfig()
+		}
+	}
+
+	if len(cfg.Repositories) == 0 {
+		infoln("No config file; starting with empty defaults")
+
+		cfg, err = readConfigXML(nil)
+		cfg.Repositories = []RepositoryConfiguration{
+			{
+				ID:        "default",
+				Directory: filepath.Join(getHomeDir(), "Sync"),
+				Nodes: []NodeConfiguration{
+					{NodeID: myID, Addresses: []string{"dynamic"}},
+				},
+			},
+		}
+
+		saveConfig()
+		infof("Edit %s to taste or use the GUI\n", cfgFile)
+	}
+
+	if reset {
+		resetRepositories()
+		os.Exit(0)
+	}
+
+	if profiler := os.Getenv("STPROFILER"); len(profiler) > 0 {
+		go func() {
+			dlog.Println("Starting profiler on", profiler)
+			err := http.ListenAndServe(profiler, nil)
+			if err != nil {
+				dlog.Fatal(err)
+			}
+		}()
+	}
+
+	// The TLS configuration is used for both the listening socket and outgoing
+	// connections.
+
+	tlsCfg := &tls.Config{
+		Certificates:           []tls.Certificate{cert},
+		NextProtos:             []string{"bep/1.0"},
+		ServerName:             myID,
+		ClientAuth:             tls.RequestClientCert,
+		SessionTicketsDisabled: true,
+		InsecureSkipVerify:     true,
+		MinVersion:             tls.VersionTLS12,
+	}
+
+	// If the write rate should be limited, set up a rate limiter for it.
+	// This will be used on connections created in the connect and listen routines.
+
+	if cfg.Options.MaxSendKbps > 0 {
+		rateBucket = ratelimit.NewBucketWithRate(float64(1000*cfg.Options.MaxSendKbps), int64(5*1000*cfg.Options.MaxSendKbps))
+	}
+
+	m := NewModel(cfg.Options.MaxChangeKbps * 1000)
+
+	for i := range cfg.Repositories {
+		cfg.Repositories[i].Nodes = cleanNodeList(cfg.Repositories[i].Nodes, myID)
+		dir := expandTilde(cfg.Repositories[i].Directory)
+		ensureDir(dir, -1)
+		m.AddRepo(cfg.Repositories[i].ID, dir, cfg.Repositories[i].Nodes)
+	}
+
+	// GUI
+	if cfg.Options.GUIEnabled && cfg.Options.GUIAddress != "" {
+		addr, err := net.ResolveTCPAddr("tcp", cfg.Options.GUIAddress)
+		if err != nil {
+			warnf("Cannot start GUI on %q: %v", cfg.Options.GUIAddress, err)
+		} else {
+			var hostOpen, hostShow string
+			switch {
+			case addr.IP == nil:
+				hostOpen = "localhost"
+				hostShow = "0.0.0.0"
+			case addr.IP.IsUnspecified():
+				hostOpen = "localhost"
+				hostShow = addr.IP.String()
+			default:
+				hostOpen = addr.IP.String()
+				hostShow = hostOpen
+			}
+
+			infof("Starting web GUI on http://%s:%d/", hostShow, addr.Port)
+			startGUI(cfg.Options.GUIAddress, m)
+			if cfg.Options.StartBrowser && len(os.Getenv("STRESTART")) == 0 {
+				openURL(fmt.Sprintf("http://%s:%d", hostOpen, addr.Port))
+			}
+		}
+	}
+
+	// Walk the repository and update the local model before establishing any
+	// connections to other nodes.
+
+	infoln("Populating repository index")
+	m.LoadIndexes(confDir)
+	m.ScanRepos()
+	m.SaveIndexes(confDir)
+
+	connOpts := map[string]string{
+		"clientId":      "syncthing",
+		"clientVersion": Version,
+		"clusterHash":   clusterHash(cfg.Repositories[0].Nodes),
+	}
+
+	// Routine to connect out to configured nodes
+	disc := discovery()
+	go listenConnect(myID, disc, m, tlsCfg, connOpts)
+
+	// Routine to pull blocks from other nodes to synchronize the local
+	// repository. Does not run when we are in read only (publish only) mode.
+	if cfg.Options.ReadOnly {
+		okln("Ready to synchronize (read only; no external updates accepted)")
+		m.StartRO()
+	} else {
+		okln("Ready to synchronize (read-write)")
+		m.StartRW(cfg.Options.ParallelRequests)
+	}
+
+	select {}
+}
+
+func resetRepositories() {
+	suffix := fmt.Sprintf(".syncthing-reset-%d", time.Now().UnixNano())
+	for _, repo := range cfg.Repositories {
+		if _, err := os.Stat(repo.Directory); err == nil {
+			infof("Reset: Moving %s -> %s", repo.Directory, repo.Directory+suffix)
+			os.Rename(repo.Directory, repo.Directory+suffix)
+		}
+	}
+
+	pat := filepath.Join(confDir, "*.idx.gz")
+	idxs, err := filepath.Glob(pat)
+	if err == nil {
+		for _, idx := range idxs {
+			infof("Reset: Removing %s", idx)
+			os.Remove(idx)
+		}
+	}
+}
+
+func restart() {
+	infoln("Restarting")
+	if os.Getenv("SMF_FMRI") != "" || os.Getenv("STNORESTART") != "" {
+		// Solaris SMF
+		infoln("Service manager detected; exit instead of restart")
+		os.Exit(0)
+	}
+
+	env := os.Environ()
+	if len(os.Getenv("STRESTART")) == 0 {
+		env = append(env, "STRESTART=1")
+	}
+	pgm, err := exec.LookPath(os.Args[0])
+	if err != nil {
+		warnln(err)
+		return
+	}
+	proc, err := os.StartProcess(pgm, os.Args, &os.ProcAttr{
+		Env:   env,
+		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
+	})
+	if err != nil {
+		fatalln(err)
+	}
+	proc.Release()
+	os.Exit(0)
+}
+
+var saveConfigCh = make(chan struct{})
+
+func saveConfigLoop(cfgFile string) {
+	for _ = range saveConfigCh {
+		fd, err := os.Create(cfgFile + ".tmp")
+		if err != nil {
+			warnln(err)
+			continue
+		}
+
+		err = writeConfigXML(fd, cfg)
+		if err != nil {
+			warnln(err)
+			fd.Close()
+			continue
+		}
+
+		err = fd.Close()
+		if err != nil {
+			warnln(err)
+			continue
+		}
+
+		err = Rename(cfgFile+".tmp", cfgFile)
+		if err != nil {
+			warnln(err)
+		}
+	}
+}
+
+func saveConfig() {
+	saveConfigCh <- struct{}{}
+}
+
+func listenConnect(myID string, disc *discover.Discoverer, m *Model, tlsCfg *tls.Config, connOpts map[string]string) {
+	var conns = make(chan *tls.Conn)
+
+	// Listen
+	for _, addr := range cfg.Options.ListenAddress {
+		addr := addr
+		go func() {
+			if debugNet {
+				dlog.Println("listening on", addr)
+			}
+			l, err := tls.Listen("tcp", addr, tlsCfg)
+			fatalErr(err)
+
+			for {
+				conn, err := l.Accept()
+				if err != nil {
+					warnln(err)
+					continue
+				}
+
+				if debugNet {
+					dlog.Println("connect from", conn.RemoteAddr())
+				}
+
+				tc := conn.(*tls.Conn)
+				err = tc.Handshake()
+				if err != nil {
+					warnln(err)
+					tc.Close()
+					continue
+				}
+
+				conns <- tc
+			}
+		}()
+	}
+
+	// Connect
+	go func() {
+		for {
+		nextNode:
+			for _, nodeCfg := range cfg.Repositories[0].Nodes {
+				if nodeCfg.NodeID == myID {
+					continue
+				}
+				if m.ConnectedTo(nodeCfg.NodeID) {
+					continue
+				}
+				for _, addr := range nodeCfg.Addresses {
+					if addr == "dynamic" {
+						if disc != nil {
+							t := disc.Lookup(nodeCfg.NodeID)
+							if len(t) == 0 {
+								continue
+							}
+							addr = t[0] //XXX: Handle all of them
+						}
+					}
+
+					if debugNet {
+						dlog.Println("dial", nodeCfg.NodeID, addr)
+					}
+					conn, err := tls.Dial("tcp", addr, tlsCfg)
+					if err != nil {
+						if debugNet {
+							dlog.Println(err)
+						}
+						continue
+					}
+
+					conns <- conn
+					continue nextNode
+				}
+			}
+
+			time.Sleep(time.Duration(cfg.Options.ReconnectIntervalS) * time.Second)
+		}
+	}()
+
+next:
+	for conn := range conns {
+		certs := conn.ConnectionState().PeerCertificates
+		if l := len(certs); l != 1 {
+			warnf("Got peer certificate list of length %d != 1; protocol error", l)
+			conn.Close()
+			continue
+		}
+		remoteID := certID(certs[0].Raw)
+
+		if remoteID == myID {
+			warnf("Connected to myself (%s) - should not happen", remoteID)
+			conn.Close()
+			continue
+		}
+
+		if m.ConnectedTo(remoteID) {
+			warnf("Connected to already connected node (%s)", remoteID)
+			conn.Close()
+			continue
+		}
+
+		for _, nodeCfg := range cfg.Repositories[0].Nodes {
+			if nodeCfg.NodeID == remoteID {
+				var wr io.Writer = conn
+				if rateBucket != nil {
+					wr = &limitedWriter{conn, rateBucket}
+				}
+				protoConn := protocol.NewConnection(remoteID, conn, wr, m, connOpts)
+				m.AddConnection(conn, protoConn)
+				continue next
+			}
+		}
+		conn.Close()
+	}
+}
+
+func discovery() *discover.Discoverer {
+	if !cfg.Options.LocalAnnEnabled {
+		return nil
+	}
+
+	infoln("Sending local discovery announcements")
+
+	if !cfg.Options.GlobalAnnEnabled {
+		cfg.Options.GlobalAnnServer = ""
+	} else {
+		infoln("Sending external discovery announcements")
+	}
+
+	disc, err := discover.NewDiscoverer(myID, cfg.Options.ListenAddress, cfg.Options.GlobalAnnServer)
+
+	if err != nil {
+		warnf("No discovery possible (%v)", err)
+	}
+
+	return disc
+}
+
+func ensureDir(dir string, mode int) {
+	fi, err := os.Stat(dir)
+	if os.IsNotExist(err) {
+		err := os.MkdirAll(dir, 0700)
+		fatalErr(err)
+	} else if mode >= 0 && err == nil && int(fi.Mode()&0777) != mode {
+		err := os.Chmod(dir, os.FileMode(mode))
+		fatalErr(err)
+	}
+}
+
+func expandTilde(p string) string {
+	if runtime.GOOS == "windows" {
+		return p
+	}
+
+	if strings.HasPrefix(p, "~/") {
+		return strings.Replace(p, "~", getUnixHomeDir(), 1)
+	}
+	return p
+}
+
+func getUnixHomeDir() string {
+	home := os.Getenv("HOME")
+	if home == "" {
+		fatalln("No home directory?")
+	}
+	return home
+}
+
+func getHomeDir() string {
+	if runtime.GOOS == "windows" {
+		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
+		if home == "" {
+			home = os.Getenv("USERPROFILE")
+		}
+		return home
+	}
+	return getUnixHomeDir()
+}
+
+func getDefaultConfDir() string {
+	if runtime.GOOS == "windows" {
+		return filepath.Join(os.Getenv("AppData"), "syncthing")
+	}
+	return expandTilde("~/.syncthing")
+}

cmd/syncthing/model.go

@@ -0,0 +1,678 @@
+package main
+
+import (
+	"compress/gzip"
+	"crypto/sha1"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/calmh/syncthing/buffers"
+	"github.com/calmh/syncthing/cid"
+	"github.com/calmh/syncthing/files"
+	"github.com/calmh/syncthing/lamport"
+	"github.com/calmh/syncthing/protocol"
+	"github.com/calmh/syncthing/scanner"
+)
+
+type Model struct {
+	repoDirs  map[string]string     // repo -> dir
+	repoFiles map[string]*files.Set // repo -> files
+	repoNodes map[string][]string   // repo -> nodeIDs
+	nodeRepos map[string][]string   // nodeID -> repos
+	rmut      sync.RWMutex          // protects the above
+
+	cm *cid.Map
+
+	protoConn map[string]protocol.Connection
+	rawConn   map[string]io.Closer
+	pmut      sync.RWMutex // protects protoConn and rawConn
+
+	sup suppressor
+
+	addedRepo bool
+	started   bool
+}
+
+var (
+	ErrNoSuchFile = errors.New("no such file")
+	ErrInvalid    = errors.New("file is invalid")
+)
+
+// NewModel creates and starts a new model. The model starts in read-only mode,
+// where it sends index information to connected peers and responds to requests
+// for file data without altering the local repository in any way.
+func NewModel(maxChangeBw int) *Model {
+	m := &Model{
+		repoDirs:  make(map[string]string),
+		repoFiles: make(map[string]*files.Set),
+		repoNodes: make(map[string][]string),
+		nodeRepos: make(map[string][]string),
+		cm:        cid.NewMap(),
+		protoConn: make(map[string]protocol.Connection),
+		rawConn:   make(map[string]io.Closer),
+		sup:       suppressor{threshold: int64(maxChangeBw)},
+	}
+
+	go m.broadcastIndexLoop()
+	return m
+}
+
+// StartRW starts read/write processing on the current model. When in
+// read/write mode the model will attempt to keep in sync with the cluster by
+// pulling needed files from peer nodes.
+func (m *Model) StartRW(threads int) {
+	m.rmut.Lock()
+	defer m.rmut.Unlock()
+
+	if !m.addedRepo {
+		panic("cannot start without repo")
+	}
+	m.started = true
+	for repo, dir := range m.repoDirs {
+		newPuller(repo, dir, m, threads)
+	}
+}
+
+// StartRO starts read only processing on the current model. When in
+// read only mode the model will announce files to the cluster but not
+// pull in any external changes.
+func (m *Model) StartRO() {
+	m.rmut.Lock()
+	defer m.rmut.Unlock()
+
+	if !m.addedRepo {
+		panic("cannot start without repo")
+	}
+	m.started = true
+	for repo, dir := range m.repoDirs {
+		newPuller(repo, dir, m, 0) // zero threads => read only
+	}
+}
+
+type ConnectionInfo struct {
+	protocol.Statistics
+	Address       string
+	ClientID      string
+	ClientVersion string
+	Completion    int
+}
+
+// ConnectionStats returns a map with connection statistics for each connected node.
+func (m *Model) ConnectionStats() map[string]ConnectionInfo {
+	type remoteAddrer interface {
+		RemoteAddr() net.Addr
+	}
+
+	m.pmut.RLock()
+	m.rmut.RLock()
+
+	var res = make(map[string]ConnectionInfo)
+	for node, conn := range m.protoConn {
+		ci := ConnectionInfo{
+			Statistics:    conn.Statistics(),
+			ClientID:      conn.Option("clientId"),
+			ClientVersion: conn.Option("clientVersion"),
+		}
+		if nc, ok := m.rawConn[node].(remoteAddrer); ok {
+			ci.Address = nc.RemoteAddr().String()
+		}
+
+		var tot int64
+		var have int64
+
+		for _, repo := range m.nodeRepos[node] {
+			for _, f := range m.repoFiles[repo].Global() {
+				if f.Flags&protocol.FlagDeleted == 0 {
+					tot += f.Size
+					have += f.Size
+				}
+			}
+
+			for _, f := range m.repoFiles[repo].Need(m.cm.Get(node)) {
+				if f.Flags&protocol.FlagDeleted == 0 {
+					have -= f.Size
+				}
+			}
+		}
+
+		ci.Completion = 100
+		if tot != 0 {
+			ci.Completion = int(100 * have / tot)
+		}
+
+		res[node] = ci
+	}
+
+	m.rmut.RUnlock()
+	m.pmut.RUnlock()
+
+	return res
+}
+
+func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) {
+	for _, f := range fs {
+		if f.Flags&protocol.FlagDeleted == 0 {
+			files++
+			bytes += f.Size
+		} else {
+			deleted++
+		}
+	}
+	return
+}
+
+// GlobalSize returns the number of files, deleted files and total bytes for all
+// files in the global model.
+func (m *Model) GlobalSize() (files, deleted int, bytes int64) {
+	m.rmut.RLock()
+	var fs []scanner.File
+	for _, rf := range m.repoFiles {
+		fs = append(fs, rf.Global()...)
+	}
+	m.rmut.RUnlock()
+	return sizeOf(fs)
+}
+
+// LocalSize returns the number of files, deleted files and total bytes for all
+// files in the local repository.
+func (m *Model) LocalSize() (files, deleted int, bytes int64) {
+	m.rmut.RLock()
+	var fs []scanner.File
+	for _, rf := range m.repoFiles {
+		fs = append(fs, rf.Have(cid.LocalID)...)
+	}
+	m.rmut.RUnlock()
+	return sizeOf(fs)
+}
+
+// InSyncSize returns the number and total byte size of the local files that
+// are in sync with the global model.
+func (m *Model) InSyncSize() (files int, bytes int64) {
+	var gf []scanner.File
+	var nf []scanner.File
+
+	m.rmut.RLock()
+	for _, rf := range m.repoFiles {
+		gf = append(gf, rf.Global()...)
+		nf = append(nf, rf.Need(cid.LocalID)...)
+	}
+	m.rmut.RUnlock()
+
+	gn, _, gb := sizeOf(gf)
+	nn, _, nb := sizeOf(nf)
+
+	return gn - nn, gb - nb
+}
+
+// NeedFiles returns the list of currently needed files and the total size.
+func (m *Model) NeedFiles() ([]scanner.File, int64) {
+	var nf []scanner.File
+	m.rmut.RLock()
+	for _, rf := range m.repoFiles {
+		nf = append(nf, rf.Need(cid.LocalID)...)
+	}
+	m.rmut.RUnlock()
+
+	var bytes int64
+	for _, f := range nf {
+		bytes += f.Size
+	}
+
+	return nf, bytes
+}
+
+// NeedFiles returns the list of currently needed files and the total size.
+func (m *Model) NeedFilesRepo(repo string) []scanner.File {
+	m.rmut.RLock()
+	nf := m.repoFiles[repo].Need(cid.LocalID)
+	m.rmut.RUnlock()
+	return nf
+}
+
+// Index is called when a new node is connected and we receive their full index.
+// Implements the protocol.Model interface.
+func (m *Model) Index(nodeID string, repo string, fs []protocol.FileInfo) {
+	if debugNet {
+		dlog.Printf("IDX(in): %s / %q: %d files", nodeID, repo, len(fs))
+	}
+
+	var files = make([]scanner.File, len(fs))
+	for i := range fs {
+		lamport.Default.Tick(fs[i].Version)
+		files[i] = fileFromFileInfo(fs[i])
+	}
+
+	id := m.cm.Get(nodeID)
+	m.rmut.RLock()
+	if r, ok := m.repoFiles[repo]; ok {
+		r.Replace(id, files)
+	} else {
+		warnf("Index from %s for nonexistant repo %q; dropping", nodeID, repo)
+	}
+	m.rmut.RUnlock()
+}
+
+// IndexUpdate is called for incremental updates to connected nodes' indexes.
+// Implements the protocol.Model interface.
+func (m *Model) IndexUpdate(nodeID string, repo string, fs []protocol.FileInfo) {
+	if debugNet {
+		dlog.Printf("IDXUP(in): %s / %q: %d files", nodeID, repo, len(fs))
+	}
+
+	var files = make([]scanner.File, len(fs))
+	for i := range fs {
+		lamport.Default.Tick(fs[i].Version)
+		files[i] = fileFromFileInfo(fs[i])
+	}
+
+	id := m.cm.Get(nodeID)
+	m.rmut.RLock()
+	if r, ok := m.repoFiles[repo]; ok {
+		r.Update(id, files)
+	} else {
+		warnf("Index update from %s for nonexistant repo %q; dropping", nodeID, repo)
+	}
+	m.rmut.RUnlock()
+}
+
+// Close removes the peer from the model and closes the underlying connection if possible.
+// Implements the protocol.Model interface.
+func (m *Model) Close(node string, err error) {
+	if debugNet {
+		dlog.Printf("%s: %v", node, err)
+	}
+	if err == protocol.ErrClusterHash {
+		warnf("Connection to %s closed due to mismatched cluster hash. Ensure that the configured cluster members are identical on both nodes.", node)
+	} else if err != io.EOF {
+		warnf("Connection to %s closed: %v", node, err)
+	}
+
+	cid := m.cm.Get(node)
+	m.rmut.RLock()
+	for _, repo := range m.nodeRepos[node] {
+		m.repoFiles[repo].Replace(cid, nil)
+	}
+	m.rmut.RUnlock()
+	m.cm.Clear(node)
+
+	m.pmut.Lock()
+	conn, ok := m.rawConn[node]
+	if ok {
+		conn.Close()
+	}
+	delete(m.protoConn, node)
+	delete(m.rawConn, node)
+	m.pmut.Unlock()
+}
+
+// Request returns the specified data segment by reading it from local disk.
+// Implements the protocol.Model interface.
+func (m *Model) Request(nodeID, repo, name string, offset int64, size int) ([]byte, error) {
+	// Verify that the requested file exists in the local model.
+	m.rmut.RLock()
+	r, ok := m.repoFiles[repo]
+	m.rmut.RUnlock()
+
+	if !ok {
+		warnf("Request from %s for file %s in nonexistent repo %q", nodeID, name, repo)
+		return nil, ErrNoSuchFile
+	}
+
+	lf := r.Get(cid.LocalID, name)
+	if offset > lf.Size {
+		warnf("SECURITY (nonexistent file) REQ(in): %s: %q o=%d s=%d", nodeID, name, offset, size)
+		return nil, ErrNoSuchFile
+	}
+
+	if lf.Suppressed {
+		return nil, ErrInvalid
+	}
+
+	if debugNet && nodeID != "<local>" {
+		dlog.Printf("REQ(in): %s: %q / %q o=%d s=%d", nodeID, repo, name, offset, size)
+	}
+	m.rmut.RLock()
+	fn := filepath.Join(m.repoDirs[repo], name)
+	m.rmut.RUnlock()
+	fd, err := os.Open(fn) // XXX: Inefficient, should cache fd?
+	if err != nil {
+		return nil, err
+	}
+	defer fd.Close()
+
+	buf := buffers.Get(int(size))
+	_, err = fd.ReadAt(buf, offset)
+	if err != nil {
+		return nil, err
+	}
+
+	return buf, nil
+}
+
+// ReplaceLocal replaces the local repository index with the given list of files.
+func (m *Model) ReplaceLocal(repo string, fs []scanner.File) {
+	m.rmut.RLock()
+	m.repoFiles[repo].ReplaceWithDelete(cid.LocalID, fs)
+	m.rmut.RUnlock()
+}
+
+func (m *Model) SeedLocal(repo string, fs []protocol.FileInfo) {
+	var sfs = make([]scanner.File, len(fs))
+	for i := 0; i < len(fs); i++ {
+		lamport.Default.Tick(fs[i].Version)
+		sfs[i] = fileFromFileInfo(fs[i])
+	}
+
+	m.rmut.RLock()
+	m.repoFiles[repo].Replace(cid.LocalID, sfs)
+	m.rmut.RUnlock()
+}
+
+func (m *Model) CurrentRepoFile(repo string, file string) scanner.File {
+	m.rmut.RLock()
+	f := m.repoFiles[repo].Get(cid.LocalID, file)
+	m.rmut.RUnlock()
+	return f
+}
+
+func (m *Model) CurrentGlobalFile(repo string, file string) scanner.File {
+	m.rmut.RLock()
+	f := m.repoFiles[repo].GetGlobal(file)
+	m.rmut.RUnlock()
+	return f
+}
+
+type cFiler struct {
+	m *Model
+	r string
+}
+
+// Implements scanner.CurrentFiler
+func (cf cFiler) CurrentFile(file string) scanner.File {
+	return cf.m.CurrentRepoFile(cf.r, file)
+}
+
+// ConnectedTo returns true if we are connected to the named node.
+func (m *Model) ConnectedTo(nodeID string) bool {
+	m.pmut.RLock()
+	_, ok := m.protoConn[nodeID]
+	m.pmut.RUnlock()
+	return ok
+}
+
+// AddConnection adds a new peer connection to the model. An initial index will
+// be sent to the connected peer, thereafter index updates whenever the local
+// repository changes.
+func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection) {
+	nodeID := protoConn.ID()
+	m.pmut.Lock()
+	if _, ok := m.protoConn[nodeID]; ok {
+		panic("add existing node")
+	}
+	m.protoConn[nodeID] = protoConn
+	if _, ok := m.rawConn[nodeID]; ok {
+		panic("add existing node")
+	}
+	m.rawConn[nodeID] = rawConn
+	m.pmut.Unlock()
+
+	go func() {
+		m.rmut.RLock()
+		repos := m.nodeRepos[nodeID]
+		m.rmut.RUnlock()
+		for _, repo := range repos {
+			idx := m.ProtocolIndex(repo)
+			if debugNet {
+				dlog.Printf("IDX(out/initial): %s: %q: %d files", nodeID, repo, len(idx))
+			}
+			protoConn.Index(repo, idx)
+		}
+	}()
+}
+
+// ProtocolIndex returns the current local index in protocol data types.
+// Must be called with the read lock held.
+func (m *Model) ProtocolIndex(repo string) []protocol.FileInfo {
+	var index []protocol.FileInfo
+
+	m.rmut.RLock()
+	fs := m.repoFiles[repo].Have(cid.LocalID)
+	m.rmut.RUnlock()
+
+	for _, f := range fs {
+		mf := fileInfoFromFile(f)
+		if debugIdx {
+			var flagComment string
+			if mf.Flags&protocol.FlagDeleted != 0 {
+				flagComment = " (deleted)"
+			}
+			dlog.Printf("IDX(out): %q/%q m=%d f=%o%s v=%d (%d blocks)", repo, mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
+		}
+		index = append(index, mf)
+	}
+
+	return index
+}
+
+func (m *Model) updateLocal(repo string, f scanner.File) {
+	m.rmut.RLock()
+	m.repoFiles[repo].Update(cid.LocalID, []scanner.File{f})
+	m.rmut.RUnlock()
+}
+
+func (m *Model) requestGlobal(nodeID, repo, name string, offset int64, size int, hash []byte) ([]byte, error) {
+	m.pmut.RLock()
+	nc, ok := m.protoConn[nodeID]
+	m.pmut.RUnlock()
+
+	if !ok {
+		return nil, fmt.Errorf("requestGlobal: no such node: %s", nodeID)
+	}
+
+	if debugNet {
+		dlog.Printf("REQ(out): %s: %q / %q o=%d s=%d h=%x", nodeID, repo, name, offset, size, hash)
+	}
+
+	return nc.Request(repo, name, offset, size)
+}
+
+func (m *Model) broadcastIndexLoop() {
+	var lastChange = map[string]uint64{}
+	for {
+		time.Sleep(5 * time.Second)
+
+		m.pmut.RLock()
+		m.rmut.RLock()
+
+		for repo, fs := range m.repoFiles {
+			c := fs.Changes(cid.LocalID)
+			if c == lastChange[repo] {
+				continue
+			}
+			lastChange[repo] = c
+
+			idx := m.ProtocolIndex(repo)
+			m.saveIndex(repo, confDir, idx)
+
+			var indexWg sync.WaitGroup
+			for _, nodeID := range m.repoNodes[repo] {
+				if conn, ok := m.protoConn[nodeID]; ok {
+					indexWg.Add(1)
+					if debugNet {
+						dlog.Printf("IDX(out/loop): %s: %d files", nodeID, len(idx))
+					}
+					go func() {
+						conn.Index(repo, idx)
+						indexWg.Done()
+					}()
+				}
+			}
+
+			indexWg.Wait()
+		}
+
+		m.rmut.RUnlock()
+		m.pmut.RUnlock()
+	}
+}
+
+func (m *Model) AddRepo(id, dir string, nodes []NodeConfiguration) {
+	if m.started {
+		panic("cannot add repo to started model")
+	}
+	if len(id) == 0 {
+		panic("cannot add empty repo id")
+	}
+
+	m.rmut.Lock()
+	m.repoDirs[id] = dir
+	m.repoFiles[id] = files.NewSet()
+
+	m.repoNodes[id] = make([]string, len(nodes))
+	for i, node := range nodes {
+		m.repoNodes[id][i] = node.NodeID
+		m.nodeRepos[node.NodeID] = append(m.nodeRepos[node.NodeID], id)
+	}
+
+	m.addedRepo = true
+	m.rmut.Unlock()
+}
+
+func (m *Model) ScanRepos() {
+	m.rmut.RLock()
+	for repo := range m.repoDirs {
+		m.ScanRepo(repo)
+	}
+	m.rmut.RUnlock()
+}
+
+func (m *Model) ScanRepo(repo string) {
+	sup := &suppressor{threshold: int64(cfg.Options.MaxChangeKbps)}
+	w := &scanner.Walker{
+		Dir:            m.repoDirs[repo],
+		IgnoreFile:     ".stignore",
+		FollowSymlinks: cfg.Options.FollowSymlinks,
+		BlockSize:      BlockSize,
+		TempNamer:      defTempNamer,
+		Suppressor:     sup,
+		CurrentFiler:   cFiler{m, repo},
+	}
+	fs, _ := w.Walk()
+	m.ReplaceLocal(repo, fs)
+}
+
+func (m *Model) SaveIndexes(dir string) {
+	m.rmut.RLock()
+	for repo := range m.repoDirs {
+		fs := m.ProtocolIndex(repo)
+		m.saveIndex(repo, dir, fs)
+	}
+	m.rmut.RUnlock()
+}
+
+func (m *Model) LoadIndexes(dir string) {
+	m.rmut.RLock()
+	for repo := range m.repoDirs {
+		fs := m.loadIndex(repo, dir)
+		m.SeedLocal(repo, fs)
+	}
+	m.rmut.RUnlock()
+}
+
+func (m *Model) saveIndex(repo string, dir string, fs []protocol.FileInfo) {
+	id := fmt.Sprintf("%x", sha1.Sum([]byte(m.repoDirs[repo])))
+	name := id + ".idx.gz"
+	name = filepath.Join(dir, name)
+
+	idxf, err := os.Create(name + ".tmp")
+	if err != nil {
+		return
+	}
+
+	gzw := gzip.NewWriter(idxf)
+
+	protocol.IndexMessage{
+		Repository: repo,
+		Files:      fs,
+	}.EncodeXDR(gzw)
+	gzw.Close()
+	idxf.Close()
+
+	Rename(name+".tmp", name)
+}
+
+func (m *Model) loadIndex(repo string, dir string) []protocol.FileInfo {
+	id := fmt.Sprintf("%x", sha1.Sum([]byte(m.repoDirs[repo])))
+	name := id + ".idx.gz"
+	name = filepath.Join(dir, name)
+
+	idxf, err := os.Open(name)
+	if err != nil {
+		return nil
+	}
+	defer idxf.Close()
+
+	gzr, err := gzip.NewReader(idxf)
+	if err != nil {
+		return nil
+	}
+	defer gzr.Close()
+
+	var im protocol.IndexMessage
+	err = im.DecodeXDR(gzr)
+	if err != nil || im.Repository != repo {
+		return nil
+	}
+
+	return im.Files
+}
+
+func fileFromFileInfo(f protocol.FileInfo) scanner.File {
+	var blocks = make([]scanner.Block, len(f.Blocks))
+	var offset int64
+	for i, b := range f.Blocks {
+		blocks[i] = scanner.Block{
+			Offset: offset,
+			Size:   b.Size,
+			Hash:   b.Hash,
+		}
+		offset += int64(b.Size)
+	}
+	return scanner.File{
+		// Name is with native separator and normalization
+		Name:       filepath.FromSlash(f.Name),
+		Size:       offset,
+		Flags:      f.Flags &^ protocol.FlagInvalid,
+		Modified:   f.Modified,
+		Version:    f.Version,
+		Blocks:     blocks,
+		Suppressed: f.Flags&protocol.FlagInvalid != 0,
+	}
+}
+
+func fileInfoFromFile(f scanner.File) protocol.FileInfo {
+	var blocks = make([]protocol.BlockInfo, len(f.Blocks))
+	for i, b := range f.Blocks {
+		blocks[i] = protocol.BlockInfo{
+			Size: b.Size,
+			Hash: b.Hash,
+		}
+	}
+	pf := protocol.FileInfo{
+		Name:     filepath.ToSlash(f.Name),
+		Flags:    f.Flags,
+		Modified: f.Modified,
+		Version:  f.Version,
+		Blocks:   blocks,
+	}
+	if f.Suppressed {
+		pf.Flags |= protocol.FlagInvalid
+	}
+	return pf
+}

cmd/syncthing/model_test.go

@@ -0,0 +1,240 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/calmh/syncthing/cid"
+	"github.com/calmh/syncthing/protocol"
+	"github.com/calmh/syncthing/scanner"
+)
+
+var testDataExpected = map[string]scanner.File{
+	"foo": scanner.File{
+		Name:     "foo",
+		Flags:    0,
+		Modified: 0,
+		Size:     7,
+		Blocks:   []scanner.Block{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
+	},
+	"empty": scanner.File{
+		Name:     "empty",
+		Flags:    0,
+		Modified: 0,
+		Size:     0,
+		Blocks:   []scanner.Block{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
+	},
+	"bar": scanner.File{
+		Name:     "bar",
+		Flags:    0,
+		Modified: 0,
+		Size:     10,
+		Blocks:   []scanner.Block{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
+	},
+}
+
+func init() {
+	// Fix expected test data to match reality
+	for n, f := range testDataExpected {
+		fi, _ := os.Stat("testdata/" + n)
+		f.Flags = uint32(fi.Mode())
+		f.Modified = fi.ModTime().Unix()
+		testDataExpected[n] = f
+	}
+}
+
+func TestRequest(t *testing.T) {
+	m := NewModel(1e6)
+	m.AddRepo("default", "testdata", nil)
+	m.ScanRepo("default")
+
+	bs, err := m.Request("some node", "default", "foo", 0, 6)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Compare(bs, []byte("foobar")) != 0 {
+		t.Errorf("Incorrect data from request: %q", string(bs))
+	}
+
+	bs, err = m.Request("some node", "default", "../walk.go", 0, 6)
+	if err == nil {
+		t.Error("Unexpected nil error on insecure file read")
+	}
+	if bs != nil {
+		t.Errorf("Unexpected non nil data on insecure file read: %q", string(bs))
+	}
+}
+
+func genFiles(n int) []protocol.FileInfo {
+	files := make([]protocol.FileInfo, n)
+	t := time.Now().Unix()
+	for i := 0; i < n; i++ {
+		files[i] = protocol.FileInfo{
+			Name:     fmt.Sprintf("file%d", i),
+			Modified: t,
+			Blocks:   []protocol.BlockInfo{{100, []byte("some hash bytes")}},
+		}
+	}
+
+	return files
+}
+
+func BenchmarkIndex10000(b *testing.B) {
+	m := NewModel(1e6)
+	m.AddRepo("default", "testdata", nil)
+	m.ScanRepo("default")
+	files := genFiles(10000)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.Index("42", "default", files)
+	}
+}
+
+func BenchmarkIndex00100(b *testing.B) {
+	m := NewModel(1e6)
+	m.AddRepo("default", "testdata", nil)
+	m.ScanRepo("default")
+	files := genFiles(100)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.Index("42", "default", files)
+	}
+}
+
+func BenchmarkIndexUpdate10000f10000(b *testing.B) {
+	m := NewModel(1e6)
+	m.AddRepo("default", "testdata", nil)
+	m.ScanRepo("default")
+	files := genFiles(10000)
+	m.Index("42", "default", files)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.IndexUpdate("42", "default", files)
+	}
+}
+
+func BenchmarkIndexUpdate10000f00100(b *testing.B) {
+	m := NewModel(1e6)
+	m.AddRepo("default", "testdata", nil)
+	m.ScanRepo("default")
+	files := genFiles(10000)
+	m.Index("42", "default", files)
+
+	ufiles := genFiles(100)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.IndexUpdate("42", "default", ufiles)
+	}
+}
+
+func BenchmarkIndexUpdate10000f00001(b *testing.B) {
+	m := NewModel(1e6)
+	m.AddRepo("default", "testdata", nil)
+	m.ScanRepo("default")
+	files := genFiles(10000)
+	m.Index("42", "default", files)
+
+	ufiles := genFiles(1)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.IndexUpdate("42", "default", ufiles)
+	}
+}
+
+type FakeConnection struct {
+	id          string
+	requestData []byte
+}
+
+func (FakeConnection) Close() error {
+	return nil
+}
+
+func (f FakeConnection) ID() string {
+	return string(f.id)
+}
+
+func (f FakeConnection) Option(string) string {
+	return ""
+}
+
+func (FakeConnection) Index(string, []protocol.FileInfo) {}
+
+func (f FakeConnection) Request(repo, name string, offset int64, size int) ([]byte, error) {
+	return f.requestData, nil
+}
+
+func (FakeConnection) Ping() bool {
+	return true
+}
+
+func (FakeConnection) Statistics() protocol.Statistics {
+	return protocol.Statistics{}
+}
+
+func BenchmarkRequest(b *testing.B) {
+	m := NewModel(1e6)
+	m.AddRepo("default", "testdata", nil)
+	m.ScanRepo("default")
+
+	const n = 1000
+	files := make([]protocol.FileInfo, n)
+	t := time.Now().Unix()
+	for i := 0; i < n; i++ {
+		files[i] = protocol.FileInfo{
+			Name:     fmt.Sprintf("file%d", i),
+			Modified: t,
+			Blocks:   []protocol.BlockInfo{{100, []byte("some hash bytes")}},
+		}
+	}
+
+	fc := FakeConnection{
+		id:          "42",
+		requestData: []byte("some data to return"),
+	}
+	m.AddConnection(fc, fc)
+	m.Index("42", "default", files)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		data, err := m.requestGlobal("42", "default", files[i%n].Name, 0, 32, nil)
+		if err != nil {
+			b.Error(err)
+		}
+		if data == nil {
+			b.Error("nil data")
+		}
+	}
+}
+
+func TestActivityMap(t *testing.T) {
+	cm := cid.NewMap()
+	fooID := cm.Get("foo")
+	if fooID == 0 {
+		t.Fatal("ID cannot be zero")
+	}
+	barID := cm.Get("bar")
+	if barID == 0 {
+		t.Fatal("ID cannot be zero")
+	}
+
+	m := make(activityMap)
+	if node := m.leastBusyNode(1<<fooID, cm); node != "foo" {
+		t.Errorf("Incorrect least busy node %q", node)
+	}
+	if node := m.leastBusyNode(1<<barID, cm); node != "bar" {
+		t.Errorf("Incorrect least busy node %q", node)
+	}
+	if node := m.leastBusyNode(1<<fooID|1<<barID, cm); node != "foo" {
+		t.Errorf("Incorrect least busy node %q", node)
+	}
+	if node := m.leastBusyNode(1<<fooID|1<<barID, cm); node != "bar" {
+		t.Errorf("Incorrect least busy node %q", node)
+	}
+}

cmd/syncthing/openurl.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2011 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os/exec"
+	"runtime"
+)
+
+func openURL(url string) error {
+	if runtime.GOOS == "windows" {
+		return exec.Command("cmd.exe", "/C", "start "+url).Run()
+	}
+
+	if runtime.GOOS == "darwin" {
+		return exec.Command("open", url).Run()
+	}
+
+	return exec.Command("xdg-open", url).Run()
+}