Integrate public as bindata optionally (#293)

* Dropped unused codekit config

* Integrated dynamic and static bindata for public

* Ignore public bindata

* Add a general generate make task

* Integrated flexible public assets into web command

* Updated vendoring, added all missiong govendor deps

* Made the linter happy with the bindata and dynamic code

* Moved public bindata definition to modules directory

* Ignoring the new bindata path now

* Updated to the new public modules import path

* Updated public bindata command and drop the new prefix

vendor/github.com/pingcap/tidb/store/hbase/kv.go

@@ -0,0 +1,238 @@
+// Copyright 2015 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hbasekv
+
+import (
+	"fmt"
+	"math/rand"
+	"net/url"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/juju/errors"
+	"github.com/ngaut/log"
+	"github.com/pingcap/go-hbase"
+	"github.com/pingcap/go-themis"
+	"github.com/pingcap/go-themis/oracle"
+	"github.com/pingcap/go-themis/oracle/oracles"
+	"github.com/pingcap/tidb/kv"
+)
+
+const (
+	// hbaseColFamily is the hbase column family name.
+	hbaseColFamily = "f"
+	// hbaseQualifier is the hbase column name.
+	hbaseQualifier = "q"
+	// hbaseFmlAndQual is a shortcut.
+	hbaseFmlAndQual = hbaseColFamily + ":" + hbaseQualifier
+	// fix length conn pool
+	hbaseConnPoolSize = 10
+)
+
+var (
+	hbaseColFamilyBytes = []byte(hbaseColFamily)
+	hbaseQualifierBytes = []byte(hbaseQualifier)
+)
+
+var (
+	_ kv.Storage = (*hbaseStore)(nil)
+)
+
+var (
+	// ErrInvalidDSN is returned when store dsn is invalid.
+	ErrInvalidDSN = errors.New("invalid dsn")
+)
+
+type storeCache struct {
+	mu    sync.Mutex
+	cache map[string]*hbaseStore
+}
+
+var mc storeCache
+
+func init() {
+	mc.cache = make(map[string]*hbaseStore)
+	rand.Seed(time.Now().UnixNano())
+}
+
+type hbaseStore struct {
+	mu        sync.Mutex
+	uuid      string
+	storeName string
+	oracle    oracle.Oracle
+	conns     []hbase.HBaseClient
+}
+
+func (s *hbaseStore) getHBaseClient() hbase.HBaseClient {
+	// return hbase connection randomly
+	return s.conns[rand.Intn(hbaseConnPoolSize)]
+}
+
+func (s *hbaseStore) Begin() (kv.Transaction, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	hbaseCli := s.getHBaseClient()
+	t, err := themis.NewTxn(hbaseCli, s.oracle)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+	txn := newHbaseTxn(t, s.storeName)
+	return txn, nil
+}
+
+func (s *hbaseStore) GetSnapshot(ver kv.Version) (kv.Snapshot, error) {
+	hbaseCli := s.getHBaseClient()
+	t, err := themis.NewTxn(hbaseCli, s.oracle)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+	return newHbaseSnapshot(t, s.storeName), nil
+}
+
+func (s *hbaseStore) Close() error {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+
+	delete(mc.cache, s.uuid)
+
+	var err error
+	for _, conn := range s.conns {
+		err = conn.Close()
+		if err != nil {
+			log.Error(err)
+		}
+	}
+	// return last error
+	return err
+}
+
+func (s *hbaseStore) UUID() string {
+	return s.uuid
+}
+
+func (s *hbaseStore) CurrentVersion() (kv.Version, error) {
+	hbaseCli := s.getHBaseClient()
+	t, err := themis.NewTxn(hbaseCli, s.oracle)
+	if err != nil {
+		return kv.Version{Ver: 0}, errors.Trace(err)
+	}
+	defer t.Release()
+
+	return kv.Version{Ver: t.GetStartTS()}, nil
+}
+
+// Driver implements engine Driver.
+type Driver struct {
+}
+
+const (
+	tsoTypeLocal = "local"
+	tsoTypeZK    = "zk"
+
+	tsoZKPath = "/zk/tso"
+)
+
+// Open opens or creates an HBase storage with given path.
+//
+// The format of path should be 'hbase://zk1,zk2,zk3/table[?tso=local|zk]'.
+// If tso is not provided, it will use a local oracle instead. (for test only)
+func (d Driver) Open(path string) (kv.Storage, error) {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+
+	zks, tso, tableName, err := parsePath(path)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+	if tso != tsoTypeLocal && tso != tsoTypeZK {
+		return nil, errors.Trace(ErrInvalidDSN)
+	}
+
+	uuid := fmt.Sprintf("hbase-%v-%v", zks, tableName)
+	if tso == tsoTypeLocal {
+		log.Warnf("hbase: store(%s) is using local oracle(for test only)", uuid)
+	}
+	if store, ok := mc.cache[uuid]; ok {
+		return store, nil
+	}
+
+	// create buffered HBase connections, HBaseClient is goroutine-safe, so
+	// it's OK to redistribute to transactions.
+	conns := make([]hbase.HBaseClient, 0, hbaseConnPoolSize)
+	for i := 0; i < hbaseConnPoolSize; i++ {
+		var c hbase.HBaseClient
+		c, err = hbase.NewClient(strings.Split(zks, ","), "/hbase")
+		if err != nil {
+			return nil, errors.Trace(err)
+		}
+		conns = append(conns, c)
+	}
+
+	c := conns[0]
+	var b bool
+	b, err = c.TableExists(tableName)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+	if !b {
+		// Create new hbase table for store.
+		t := hbase.NewTableDesciptor(tableName)
+		cf := hbase.NewColumnFamilyDescriptor(hbaseColFamily)
+		cf.AddAttr("THEMIS_ENABLE", "true")
+		t.AddColumnDesc(cf)
+		//TODO: specify split?
+		if err := c.CreateTable(t, nil); err != nil {
+			return nil, errors.Trace(err)
+		}
+	}
+
+	var ora oracle.Oracle
+	switch tso {
+	case tsoTypeLocal:
+		ora = oracles.NewLocalOracle()
+	case tsoTypeZK:
+		ora = oracles.NewRemoteOracle(zks, tsoZKPath)
+	}
+
+	s := &hbaseStore{
+		uuid:      uuid,
+		storeName: tableName,
+		oracle:    ora,
+		conns:     conns,
+	}
+	mc.cache[uuid] = s
+	return s, nil
+}
+
+func parsePath(path string) (zks, tso, tableName string, err error) {
+	u, err := url.Parse(path)
+	if err != nil {
+		return "", "", "", errors.Trace(err)
+	}
+	if strings.ToLower(u.Scheme) != "hbase" {
+		return "", "", "", errors.Trace(ErrInvalidDSN)
+	}
+	p, tableName := filepath.Split(u.Path)
+	if p != "/" {
+		return "", "", "", errors.Trace(ErrInvalidDSN)
+	}
+	zks = u.Host
+	tso = u.Query().Get("tso")
+	if tso == "" {
+		tso = tsoTypeLocal
+	}
+	return zks, tso, tableName, nil
+}

vendor/github.com/pingcap/tidb/store/hbase/snapshot.go

@@ -0,0 +1,143 @@
+// Copyright 2015 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hbasekv
+
+import (
+	"github.com/juju/errors"
+	"github.com/pingcap/go-hbase"
+	"github.com/pingcap/go-themis"
+	"github.com/pingcap/tidb/kv"
+)
+
+var (
+	_ kv.Snapshot = (*hbaseSnapshot)(nil)
+	_ kv.Iterator = (*hbaseIter)(nil)
+)
+
+// hbaseBatchSize is used for go-themis Scanner.
+const hbaseBatchSize = 1000
+
+// hbaseSnapshot implements MvccSnapshot interface.
+type hbaseSnapshot struct {
+	txn       themis.Txn
+	storeName string
+}
+
+// newHBaseSnapshot creates a snapshot of an HBase store.
+func newHbaseSnapshot(txn themis.Txn, storeName string) *hbaseSnapshot {
+	return &hbaseSnapshot{
+		txn:       txn,
+		storeName: storeName,
+	}
+}
+
+// Get gets the value for key k from snapshot.
+func (s *hbaseSnapshot) Get(k kv.Key) ([]byte, error) {
+	g := hbase.NewGet([]byte(k))
+	g.AddColumn(hbaseColFamilyBytes, hbaseQualifierBytes)
+	v, err := internalGet(s, g)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+	return v, nil
+}
+
+// BatchGet implements kv.Snapshot.BatchGet interface.
+func (s *hbaseSnapshot) BatchGet(keys []kv.Key) (map[string][]byte, error) {
+	gets := make([]*hbase.Get, len(keys))
+	for i, key := range keys {
+		g := hbase.NewGet(key)
+		g.AddColumn(hbaseColFamilyBytes, hbaseQualifierBytes)
+		gets[i] = g
+	}
+	rows, err := s.txn.Gets(s.storeName, gets)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+
+	m := make(map[string][]byte, len(rows))
+	for _, r := range rows {
+		k := string(r.Row)
+		v := r.Columns[hbaseFmlAndQual].Value
+		m[k] = v
+	}
+	return m, nil
+}
+
+func internalGet(s *hbaseSnapshot, g *hbase.Get) ([]byte, error) {
+	r, err := s.txn.Get(s.storeName, g)
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+	if r == nil || len(r.Columns) == 0 {
+		return nil, errors.Trace(kv.ErrNotExist)
+	}
+	return r.Columns[hbaseFmlAndQual].Value, nil
+}
+
+func (s *hbaseSnapshot) Seek(k kv.Key) (kv.Iterator, error) {
+	scanner := s.txn.GetScanner([]byte(s.storeName), []byte(k), nil, hbaseBatchSize)
+	return newInnerScanner(scanner), nil
+}
+
+func newInnerScanner(scanner *themis.ThemisScanner) kv.Iterator {
+	it := &hbaseIter{
+		ThemisScanner: scanner,
+	}
+	it.Next()
+	return it
+}
+
+func (s *hbaseSnapshot) Release() {
+	if s.txn != nil {
+		s.txn.Release()
+		s.txn = nil
+	}
+}
+
+type hbaseIter struct {
+	*themis.ThemisScanner
+	rs *hbase.ResultRow
+}
+
+func (it *hbaseIter) Next() error {
+	it.rs = it.ThemisScanner.Next()
+	return nil
+}
+
+func (it *hbaseIter) Valid() bool {
+	if it.rs == nil || len(it.rs.Columns) == 0 {
+		return false
+	}
+	if it.ThemisScanner.Closed() {
+		return false
+	}
+	return true
+}
+
+func (it *hbaseIter) Key() kv.Key {
+	return it.rs.Row
+}
+
+func (it *hbaseIter) Value() []byte {
+	return it.rs.Columns[hbaseFmlAndQual].Value
+}
+
+func (it *hbaseIter) Close() {
+	if it.ThemisScanner != nil {
+		it.ThemisScanner.Close()
+		it.ThemisScanner = nil
+	}
+	it.rs = nil
+}

vendor/github.com/pingcap/tidb/store/hbase/txn.go

@@ -0,0 +1,182 @@
+// Copyright 2015 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hbasekv
+
+import (
+	"fmt"
+
+	"github.com/juju/errors"
+	"github.com/ngaut/log"
+	"github.com/pingcap/go-hbase"
+	"github.com/pingcap/go-themis"
+	"github.com/pingcap/tidb/kv"
+)
+
+var (
+	_ kv.Transaction = (*hbaseTxn)(nil)
+)
+
+// dbTxn implements kv.Transacton. It is not thread safe.
+type hbaseTxn struct {
+	us        kv.UnionStore
+	txn       themis.Txn
+	store     *hbaseStore // for commit
+	storeName string
+	tid       uint64
+	valid     bool
+	version   kv.Version // commit version
+	dirty     bool
+}
+
+func newHbaseTxn(t themis.Txn, storeName string) *hbaseTxn {
+	return &hbaseTxn{
+		txn:       t,
+		valid:     true,
+		storeName: storeName,
+		tid:       t.GetStartTS(),
+		us:        kv.NewUnionStore(newHbaseSnapshot(t, storeName)),
+	}
+}
+
+// Implement transaction interface
+
+func (txn *hbaseTxn) Get(k kv.Key) ([]byte, error) {
+	log.Debugf("[kv] get key:%q, txn:%d", k, txn.tid)
+	return txn.us.Get(k)
+}
+
+func (txn *hbaseTxn) Set(k kv.Key, v []byte) error {
+	log.Debugf("[kv] set %q txn:%d", k, txn.tid)
+	txn.dirty = true
+	return txn.us.Set(k, v)
+}
+
+func (txn *hbaseTxn) String() string {
+	return fmt.Sprintf("%d", txn.tid)
+}
+
+func (txn *hbaseTxn) Seek(k kv.Key) (kv.Iterator, error) {
+	log.Debugf("[kv] seek %q txn:%d", k, txn.tid)
+	return txn.us.Seek(k)
+}
+
+func (txn *hbaseTxn) Delete(k kv.Key) error {
+	log.Debugf("[kv] delete %q txn:%d", k, txn.tid)
+	txn.dirty = true
+	return txn.us.Delete(k)
+}
+
+func (txn *hbaseTxn) SetOption(opt kv.Option, val interface{}) {
+	txn.us.SetOption(opt, val)
+}
+
+func (txn *hbaseTxn) DelOption(opt kv.Option) {
+	txn.us.DelOption(opt)
+}
+
+func (txn *hbaseTxn) doCommit() error {
+	if err := txn.us.CheckLazyConditionPairs(); err != nil {
+		return errors.Trace(err)
+	}
+
+	err := txn.us.WalkBuffer(func(k kv.Key, v []byte) error {
+		row := append([]byte(nil), k...)
+		if len(v) == 0 { // Deleted marker
+			d := hbase.NewDelete(row)
+			d.AddStringColumn(hbaseColFamily, hbaseQualifier)
+			err := txn.txn.Delete(txn.storeName, d)
+			if err != nil {
+				return errors.Trace(err)
+			}
+		} else {
+			val := append([]byte(nil), v...)
+			p := hbase.NewPut(row)
+			p.AddValue(hbaseColFamilyBytes, hbaseQualifierBytes, val)
+			txn.txn.Put(txn.storeName, p)
+		}
+		return nil
+	})
+
+	if err != nil {
+		return errors.Trace(err)
+	}
+
+	err = txn.txn.Commit()
+	if err != nil {
+		log.Error(err)
+		return errors.Trace(err)
+	}
+
+	txn.version = kv.NewVersion(txn.txn.GetCommitTS())
+	log.Debugf("[kv] commit successfully, txn.version:%d", txn.version.Ver)
+	return nil
+}
+
+func (txn *hbaseTxn) Commit() error {
+	if !txn.valid {
+		return kv.ErrInvalidTxn
+	}
+	log.Debugf("[kv] start to commit txn %d", txn.tid)
+	defer func() {
+		txn.close()
+	}()
+	return txn.doCommit()
+}
+
+func (txn *hbaseTxn) close() error {
+	txn.us.Release()
+	txn.valid = false
+	return nil
+}
+
+//if fail, themis auto rollback
+func (txn *hbaseTxn) Rollback() error {
+	if !txn.valid {
+		return kv.ErrInvalidTxn
+	}
+	log.Warnf("[kv] Rollback txn %d", txn.tid)
+	return txn.close()
+}
+
+func (txn *hbaseTxn) LockKeys(keys ...kv.Key) error {
+	for _, key := range keys {
+		if err := txn.txn.LockRow(txn.storeName, key); err != nil {
+			return errors.Trace(err)
+		}
+	}
+	return nil
+}
+
+func (txn *hbaseTxn) IsReadOnly() bool {
+	return !txn.dirty
+}
+
+func (txn *hbaseTxn) StartTS() int64 {
+	return int64(txn.tid)
+}
+
+func (txn *hbaseTxn) GetClient() kv.Client {
+	return nil
+}
+
+type hbaseClient struct {
+}
+
+func (c *hbaseClient) SupportRequestType(reqType, subType int64) bool {
+	return false
+}
+
+func (c *hbaseClient) Send(req *kv.Request) kv.Response {
+	return nil
+}