Integrate public as bindata optionally (#293)

* Dropped unused codekit config

* Integrated dynamic and static bindata for public

* Ignore public bindata

* Add a general generate make task

* Integrated flexible public assets into web command

* Updated vendoring, added all missiong govendor deps

* Made the linter happy with the bindata and dynamic code

* Moved public bindata definition to modules directory

* Ignoring the new bindata path now

* Updated to the new public modules import path

* Updated public bindata command and drop the new prefix

vendor/github.com/ngaut/pools/id_pool.go

@@ -0,0 +1,72 @@
+// Copyright 2014, Google Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package pools
+
+import (
+	"fmt"
+	"sync"
+)
+
+// IDPool is used to ensure that the set of IDs in use concurrently never
+// contains any duplicates. The IDs start at 1 and increase without bound, but
+// will never be larger than the peak number of concurrent uses.
+//
+// IDPool's Get() and Set() methods can be used concurrently.
+type IDPool struct {
+	sync.Mutex
+
+	// used holds the set of values that have been returned to us with Put().
+	used map[uint32]bool
+	// maxUsed remembers the largest value we've given out.
+	maxUsed uint32
+}
+
+// NewIDPool creates and initializes an IDPool.
+func NewIDPool() *IDPool {
+	return &IDPool{
+		used: make(map[uint32]bool),
+	}
+}
+
+// Get returns an ID that is unique among currently active users of this pool.
+func (pool *IDPool) Get() (id uint32) {
+	pool.Lock()
+	defer pool.Unlock()
+
+	// Pick a value that's been returned, if any.
+	for key, _ := range pool.used {
+		delete(pool.used, key)
+		return key
+	}
+
+	// No recycled IDs are available, so increase the pool size.
+	pool.maxUsed += 1
+	return pool.maxUsed
+}
+
+// Put recycles an ID back into the pool for others to use. Putting back a value
+// or 0, or a value that is not currently "checked out", will result in a panic
+// because that should never happen except in the case of a programming error.
+func (pool *IDPool) Put(id uint32) {
+	pool.Lock()
+	defer pool.Unlock()
+
+	if id < 1 || id > pool.maxUsed {
+		panic(fmt.Errorf("IDPool.Put(%v): invalid value, must be in the range [1,%v]", id, pool.maxUsed))
+	}
+
+	if pool.used[id] {
+		panic(fmt.Errorf("IDPool.Put(%v): can't put value that was already recycled", id))
+	}
+
+	// If we're recycling maxUsed, just shrink the pool.
+	if id == pool.maxUsed {
+		pool.maxUsed = id - 1
+		return
+	}
+
+	// Add it to the set of recycled IDs.
+	pool.used[id] = true
+}

vendor/github.com/ngaut/pools/numbered.go

@@ -0,0 +1,149 @@
+// Copyright 2012, Google Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package pools
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+// Numbered allows you to manage resources by tracking them with numbers.
+// There are no interface restrictions on what you can track.
+type Numbered struct {
+	mu        sync.Mutex
+	empty     *sync.Cond // Broadcast when pool becomes empty
+	resources map[int64]*numberedWrapper
+}
+
+type numberedWrapper struct {
+	val         interface{}
+	inUse       bool
+	purpose     string
+	timeCreated time.Time
+	timeUsed    time.Time
+}
+
+func NewNumbered() *Numbered {
+	n := &Numbered{resources: make(map[int64]*numberedWrapper)}
+	n.empty = sync.NewCond(&n.mu)
+	return n
+}
+
+// Register starts tracking a resource by the supplied id.
+// It does not lock the object.
+// It returns an error if the id already exists.
+func (nu *Numbered) Register(id int64, val interface{}) error {
+	nu.mu.Lock()
+	defer nu.mu.Unlock()
+	if _, ok := nu.resources[id]; ok {
+		return fmt.Errorf("already present")
+	}
+	now := time.Now()
+	nu.resources[id] = &numberedWrapper{
+		val:         val,
+		timeCreated: now,
+		timeUsed:    now,
+	}
+	return nil
+}
+
+// Unregiester forgets the specified resource.
+// If the resource is not present, it's ignored.
+func (nu *Numbered) Unregister(id int64) {
+	nu.mu.Lock()
+	defer nu.mu.Unlock()
+	delete(nu.resources, id)
+	if len(nu.resources) == 0 {
+		nu.empty.Broadcast()
+	}
+}
+
+// Get locks the resource for use. It accepts a purpose as a string.
+// If it cannot be found, it returns a "not found" error. If in use,
+// it returns a "in use: purpose" error.
+func (nu *Numbered) Get(id int64, purpose string) (val interface{}, err error) {
+	nu.mu.Lock()
+	defer nu.mu.Unlock()
+	nw, ok := nu.resources[id]
+	if !ok {
+		return nil, fmt.Errorf("not found")
+	}
+	if nw.inUse {
+		return nil, fmt.Errorf("in use: %s", nw.purpose)
+	}
+	nw.inUse = true
+	nw.purpose = purpose
+	return nw.val, nil
+}
+
+// Put unlocks a resource for someone else to use.
+func (nu *Numbered) Put(id int64) {
+	nu.mu.Lock()
+	defer nu.mu.Unlock()
+	if nw, ok := nu.resources[id]; ok {
+		nw.inUse = false
+		nw.purpose = ""
+		nw.timeUsed = time.Now()
+	}
+}
+
+// GetOutdated returns a list of resources that are older than age, and locks them.
+// It does not return any resources that are already locked.
+func (nu *Numbered) GetOutdated(age time.Duration, purpose string) (vals []interface{}) {
+	nu.mu.Lock()
+	defer nu.mu.Unlock()
+	now := time.Now()
+	for _, nw := range nu.resources {
+		if nw.inUse {
+			continue
+		}
+		if nw.timeCreated.Add(age).Sub(now) <= 0 {
+			nw.inUse = true
+			nw.purpose = purpose
+			vals = append(vals, nw.val)
+		}
+	}
+	return vals
+}
+
+// GetIdle returns a list of resurces that have been idle for longer
+// than timeout, and locks them. It does not return any resources that
+// are already locked.
+func (nu *Numbered) GetIdle(timeout time.Duration, purpose string) (vals []interface{}) {
+	nu.mu.Lock()
+	defer nu.mu.Unlock()
+	now := time.Now()
+	for _, nw := range nu.resources {
+		if nw.inUse {
+			continue
+		}
+		if nw.timeUsed.Add(timeout).Sub(now) <= 0 {
+			nw.inUse = true
+			nw.purpose = purpose
+			vals = append(vals, nw.val)
+		}
+	}
+	return vals
+}
+
+// WaitForEmpty returns as soon as the pool becomes empty
+func (nu *Numbered) WaitForEmpty() {
+	nu.mu.Lock()
+	defer nu.mu.Unlock()
+	for len(nu.resources) != 0 {
+		nu.empty.Wait()
+	}
+}
+
+func (nu *Numbered) StatsJSON() string {
+	return fmt.Sprintf("{\"Size\": %v}", nu.Size())
+}
+
+func (nu *Numbered) Size() (size int64) {
+	nu.mu.Lock()
+	defer nu.mu.Unlock()
+	return int64(len(nu.resources))
+}

vendor/github.com/ngaut/pools/resource_pool.go

@@ -0,0 +1,228 @@
+// Copyright 2012, Google Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package pools provides functionality to manage and reuse resources
+// like connections.
+package pools
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/ngaut/sync2"
+)
+
+var (
+	CLOSED_ERR = fmt.Errorf("ResourcePool is closed")
+)
+
+// Factory is a function that can be used to create a resource.
+type Factory func() (Resource, error)
+
+// Every resource needs to suport the Resource interface.
+// Thread synchronization between Close() and IsClosed()
+// is the responsibility the caller.
+type Resource interface {
+	Close()
+}
+
+// ResourcePool allows you to use a pool of resources.
+type ResourcePool struct {
+	resources   chan resourceWrapper
+	factory     Factory
+	capacity    sync2.AtomicInt64
+	idleTimeout sync2.AtomicDuration
+
+	// stats
+	waitCount sync2.AtomicInt64
+	waitTime  sync2.AtomicDuration
+}
+
+type resourceWrapper struct {
+	resource Resource
+	timeUsed time.Time
+}
+
+// NewResourcePool creates a new ResourcePool pool.
+// capacity is the initial capacity of the pool.
+// maxCap is the maximum capacity.
+// If a resource is unused beyond idleTimeout, it's discarded.
+// An idleTimeout of 0 means that there is no timeout.
+func NewResourcePool(factory Factory, capacity, maxCap int, idleTimeout time.Duration) *ResourcePool {
+	if capacity <= 0 || maxCap <= 0 || capacity > maxCap {
+		panic(fmt.Errorf("Invalid/out of range capacity"))
+	}
+	rp := &ResourcePool{
+		resources:   make(chan resourceWrapper, maxCap),
+		factory:     factory,
+		capacity:    sync2.AtomicInt64(capacity),
+		idleTimeout: sync2.AtomicDuration(idleTimeout),
+	}
+	for i := 0; i < capacity; i++ {
+		rp.resources <- resourceWrapper{}
+	}
+	return rp
+}
+
+// Close empties the pool calling Close on all its resources.
+// You can call Close while there are outstanding resources.
+// It waits for all resources to be returned (Put).
+// After a Close, Get and TryGet are not allowed.
+func (rp *ResourcePool) Close() {
+	rp.SetCapacity(0)
+}
+
+func (rp *ResourcePool) IsClosed() (closed bool) {
+	return rp.capacity.Get() == 0
+}
+
+// Get will return the next available resource. If capacity
+// has not been reached, it will create a new one using the factory. Otherwise,
+// it will indefinitely wait till the next resource becomes available.
+func (rp *ResourcePool) Get() (resource Resource, err error) {
+	return rp.get(true)
+}
+
+// TryGet will return the next available resource. If none is available, and capacity
+// has not been reached, it will create a new one using the factory. Otherwise,
+// it will return nil with no error.
+func (rp *ResourcePool) TryGet() (resource Resource, err error) {
+	return rp.get(false)
+}
+
+func (rp *ResourcePool) get(wait bool) (resource Resource, err error) {
+	// Fetch
+	var wrapper resourceWrapper
+	var ok bool
+	select {
+	case wrapper, ok = <-rp.resources:
+	default:
+		if !wait {
+			return nil, nil
+		}
+		startTime := time.Now()
+		wrapper, ok = <-rp.resources
+		rp.recordWait(startTime)
+	}
+	if !ok {
+		return nil, CLOSED_ERR
+	}
+
+	// Unwrap
+	timeout := rp.idleTimeout.Get()
+	if wrapper.resource != nil && timeout > 0 && wrapper.timeUsed.Add(timeout).Sub(time.Now()) < 0 {
+		wrapper.resource.Close()
+		wrapper.resource = nil
+	}
+	if wrapper.resource == nil {
+		wrapper.resource, err = rp.factory()
+		if err != nil {
+			rp.resources <- resourceWrapper{}
+		}
+	}
+	return wrapper.resource, err
+}
+
+// Put will return a resource to the pool. For every successful Get,
+// a corresponding Put is required. If you no longer need a resource,
+// you will need to call Put(nil) instead of returning the closed resource.
+// The will eventually cause a new resource to be created in its place.
+func (rp *ResourcePool) Put(resource Resource) {
+	var wrapper resourceWrapper
+	if resource != nil {
+		wrapper = resourceWrapper{resource, time.Now()}
+	}
+	select {
+	case rp.resources <- wrapper:
+	default:
+		panic(fmt.Errorf("Attempt to Put into a full ResourcePool"))
+	}
+}
+
+// SetCapacity changes the capacity of the pool.
+// You can use it to shrink or expand, but not beyond
+// the max capacity. If the change requires the pool
+// to be shrunk, SetCapacity waits till the necessary
+// number of resources are returned to the pool.
+// A SetCapacity of 0 is equivalent to closing the ResourcePool.
+func (rp *ResourcePool) SetCapacity(capacity int) error {
+	if capacity < 0 || capacity > cap(rp.resources) {
+		return fmt.Errorf("capacity %d is out of range", capacity)
+	}
+
+	// Atomically swap new capacity with old, but only
+	// if old capacity is non-zero.
+	var oldcap int
+	for {
+		oldcap = int(rp.capacity.Get())
+		if oldcap == 0 {
+			return CLOSED_ERR
+		}
+		if oldcap == capacity {
+			return nil
+		}
+		if rp.capacity.CompareAndSwap(int64(oldcap), int64(capacity)) {
+			break
+		}
+	}
+
+	if capacity < oldcap {
+		for i := 0; i < oldcap-capacity; i++ {
+			wrapper := <-rp.resources
+			if wrapper.resource != nil {
+				wrapper.resource.Close()
+			}
+		}
+	} else {
+		for i := 0; i < capacity-oldcap; i++ {
+			rp.resources <- resourceWrapper{}
+		}
+	}
+	if capacity == 0 {
+		close(rp.resources)
+	}
+	return nil
+}
+
+func (rp *ResourcePool) recordWait(start time.Time) {
+	rp.waitCount.Add(1)
+	rp.waitTime.Add(time.Now().Sub(start))
+}
+
+func (rp *ResourcePool) SetIdleTimeout(idleTimeout time.Duration) {
+	rp.idleTimeout.Set(idleTimeout)
+}
+
+func (rp *ResourcePool) StatsJSON() string {
+	c, a, mx, wc, wt, it := rp.Stats()
+	return fmt.Sprintf(`{"Capacity": %v, "Available": %v, "MaxCapacity": %v, "WaitCount": %v, "WaitTime": %v, "IdleTimeout": %v}`, c, a, mx, wc, int64(wt), int64(it))
+}
+
+func (rp *ResourcePool) Stats() (capacity, available, maxCap, waitCount int64, waitTime, idleTimeout time.Duration) {
+	return rp.Capacity(), rp.Available(), rp.MaxCap(), rp.WaitCount(), rp.WaitTime(), rp.IdleTimeout()
+}
+
+func (rp *ResourcePool) Capacity() int64 {
+	return rp.capacity.Get()
+}
+
+func (rp *ResourcePool) Available() int64 {
+	return int64(len(rp.resources))
+}
+
+func (rp *ResourcePool) MaxCap() int64 {
+	return int64(cap(rp.resources))
+}
+
+func (rp *ResourcePool) WaitCount() int64 {
+	return rp.waitCount.Get()
+}
+
+func (rp *ResourcePool) WaitTime() time.Duration {
+	return rp.waitTime.Get()
+}
+
+func (rp *ResourcePool) IdleTimeout() time.Duration {
+	return rp.idleTimeout.Get()
+}

vendor/github.com/ngaut/pools/roundrobin.go

@@ -0,0 +1,214 @@
+// Copyright 2012, Google Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package pools
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+// RoundRobin is deprecated. Use ResourcePool instead.
+// RoundRobin allows you to use a pool of resources in a round robin fashion.
+type RoundRobin struct {
+	mu          sync.Mutex
+	available   *sync.Cond
+	resources   chan fifoWrapper
+	size        int64
+	factory     Factory
+	idleTimeout time.Duration
+
+	// stats
+	waitCount int64
+	waitTime  time.Duration
+}
+
+type fifoWrapper struct {
+	resource Resource
+	timeUsed time.Time
+}
+
+// NewRoundRobin creates a new RoundRobin pool.
+// capacity is the maximum number of resources RoundRobin will create.
+// factory will be the function used to create resources.
+// If a resource is unused beyond idleTimeout, it's discarded.
+func NewRoundRobin(capacity int, idleTimeout time.Duration) *RoundRobin {
+	r := &RoundRobin{
+		resources:   make(chan fifoWrapper, capacity),
+		size:        0,
+		idleTimeout: idleTimeout,
+	}
+	r.available = sync.NewCond(&r.mu)
+	return r
+}
+
+// Open starts allowing the creation of resources
+func (rr *RoundRobin) Open(factory Factory) {
+	rr.mu.Lock()
+	defer rr.mu.Unlock()
+	rr.factory = factory
+}
+
+// Close empties the pool calling Close on all its resources.
+// It waits for all resources to be returned (Put).
+func (rr *RoundRobin) Close() {
+	rr.mu.Lock()
+	defer rr.mu.Unlock()
+	for rr.size > 0 {
+		select {
+		case fw := <-rr.resources:
+			go fw.resource.Close()
+			rr.size--
+		default:
+			rr.available.Wait()
+		}
+	}
+	rr.factory = nil
+}
+
+func (rr *RoundRobin) IsClosed() bool {
+	return rr.factory == nil
+}
+
+// Get will return the next available resource. If none is available, and capacity
+// has not been reached, it will create a new one using the factory. Otherwise,
+// it will indefinitely wait till the next resource becomes available.
+func (rr *RoundRobin) Get() (resource Resource, err error) {
+	return rr.get(true)
+}
+
+// TryGet will return the next available resource. If none is available, and capacity
+// has not been reached, it will create a new one using the factory. Otherwise,
+// it will return nil with no error.
+func (rr *RoundRobin) TryGet() (resource Resource, err error) {
+	return rr.get(false)
+}
+
+func (rr *RoundRobin) get(wait bool) (resource Resource, err error) {
+	rr.mu.Lock()
+	defer rr.mu.Unlock()
+	// Any waits in this loop will release the lock, and it will be
+	// reacquired before the waits return.
+	for {
+		select {
+		case fw := <-rr.resources:
+			// Found a free resource in the channel
+			if rr.idleTimeout > 0 && fw.timeUsed.Add(rr.idleTimeout).Sub(time.Now()) < 0 {
+				// resource has been idle for too long. Discard & go for next.
+				go fw.resource.Close()
+				rr.size--
+				// Nobody else should be waiting, but signal anyway.
+				rr.available.Signal()
+				continue
+			}
+			return fw.resource, nil
+		default:
+			// resource channel is empty
+			if rr.size >= int64(cap(rr.resources)) {
+				// The pool is full
+				if wait {
+					start := time.Now()
+					rr.available.Wait()
+					rr.recordWait(start)
+					continue
+				}
+				return nil, nil
+			}
+			// Pool is not full. Create a resource.
+			if resource, err = rr.waitForCreate(); err != nil {
+				// size was decremented, and somebody could be waiting.
+				rr.available.Signal()
+				return nil, err
+			}
+			// Creation successful. Account for this by incrementing size.
+			rr.size++
+			return resource, err
+		}
+	}
+}
+
+func (rr *RoundRobin) recordWait(start time.Time) {
+	rr.waitCount++
+	rr.waitTime += time.Now().Sub(start)
+}
+
+func (rr *RoundRobin) waitForCreate() (resource Resource, err error) {
+	// Prevent thundering herd: increment size before creating resource, and decrement after.
+	rr.size++
+	rr.mu.Unlock()
+	defer func() {
+		rr.mu.Lock()
+		rr.size--
+	}()
+	return rr.factory()
+}
+
+// Put will return a resource to the pool. You MUST return every resource to the pool,
+// even if it's closed. If a resource is closed, you should call Put(nil).
+func (rr *RoundRobin) Put(resource Resource) {
+	rr.mu.Lock()
+	defer rr.available.Signal()
+	defer rr.mu.Unlock()
+
+	if rr.size > int64(cap(rr.resources)) {
+		if resource != nil {
+			go resource.Close()
+		}
+		rr.size--
+	} else if resource == nil {
+		rr.size--
+	} else {
+		if len(rr.resources) == cap(rr.resources) {
+			panic("unexpected")
+		}
+		rr.resources <- fifoWrapper{resource, time.Now()}
+	}
+}
+
+// Set capacity changes the capacity of the pool.
+// You can use it to expand or shrink.
+func (rr *RoundRobin) SetCapacity(capacity int) error {
+	rr.mu.Lock()
+	defer rr.available.Broadcast()
+	defer rr.mu.Unlock()
+
+	nr := make(chan fifoWrapper, capacity)
+	// This loop transfers resources from the old channel
+	// to the new one, until it fills up or runs out.
+	// It discards extras, if any.
+	for {
+		select {
+		case fw := <-rr.resources:
+			if len(nr) < cap(nr) {
+				nr <- fw
+			} else {
+				go fw.resource.Close()
+				rr.size--
+			}
+			continue
+		default:
+		}
+		break
+	}
+	rr.resources = nr
+	return nil
+}
+
+func (rr *RoundRobin) SetIdleTimeout(idleTimeout time.Duration) {
+	rr.mu.Lock()
+	defer rr.mu.Unlock()
+	rr.idleTimeout = idleTimeout
+}
+
+func (rr *RoundRobin) StatsJSON() string {
+	s, c, a, wc, wt, it := rr.Stats()
+	return fmt.Sprintf("{\"Size\": %v, \"Capacity\": %v, \"Available\": %v, \"WaitCount\": %v, \"WaitTime\": %v, \"IdleTimeout\": %v}", s, c, a, wc, int64(wt), int64(it))
+}
+
+func (rr *RoundRobin) Stats() (size, capacity, available, waitCount int64, waitTime, idleTimeout time.Duration) {
+	rr.mu.Lock()
+	defer rr.mu.Unlock()
+	return rr.size, int64(cap(rr.resources)), int64(len(rr.resources)), rr.waitCount, rr.waitTime, rr.idleTimeout
+}

vendor/github.com/ngaut/pools/vitess_license

@@ -0,0 +1,28 @@
+Copyright 2012, Google Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+    * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.