Refactor: Move login out of models (#16199)

`models` does far too much. In particular it handles all `UserSignin`.

It shouldn't be responsible for calling LDAP, SMTP or PAM for signing in.

Therefore we should move this code out of `models`.

This code has to depend on `models` - therefore it belongs in `services`.

There is a package in `services` called `auth` and clearly this functionality belongs in there.

Plan:

- [x] Change `auth.Auth` to `auth.Method` - as they represent methods of authentication.
- [x] Move `models.UserSignIn` into `auth`
- [x] Move `models.ExternalUserLogin`
- [x] Move most of the `LoginVia*` methods to `auth` or subpackages
- [x] Move Resynchronize functionality to `auth`
  - Involved some restructuring of `models/ssh_key.go` to reduce the size of this massive file and simplify its files.
- [x] Move the rest of the LDAP functionality in to the ldap subpackage
- [x] Re-factor the login sources to express an interfaces `auth.Source`?
  - I've done this through some smaller interfaces Authenticator and Synchronizable - which would allow us to extend things in future
- [x] Now LDAP is out of models - need to think about modules/auth/ldap and I think all of that functionality might just be moveable
- [x] Similarly a lot Oauth2 functionality need not be in models too and should be moved to services/auth/source/oauth2
  - [x] modules/auth/oauth2/oauth2.go uses xorm... This is naughty - probably need to move this into models.
  - [x] models/oauth2.go - mostly should be in modules/auth/oauth2 or services/auth/source/oauth2 
- [x] More simplifications of login_source.go may need to be done
- Allow wiring in of notify registration -  *this can now easily be done - but I think we should do it in another PR*  - see #16178 
- More refactors...?
  - OpenID should probably become an auth Method but I think that can be left for another PR
  - Methods should also probably be cleaned up  - again another PR I think.
  - SSPI still needs more refactors.* Rename auth.Auth auth.Method
* Restructure ssh_key.go

- move functions from models/user.go that relate to ssh_key to ssh_key
- split ssh_key.go to try create clearer function domains for allow for
future refactors here.

Signed-off-by: Andrew Thornton <art27@cantab.net>

models/migrations/fixtures/Test_unwrapLDAPSourceCfg/login_source.yml

@@ -0,0 +1,48 @@
+# type LoginSource struct {
+#   ID        int64 `xorm:"pk autoincr"`
+#   Type      int
+#   Cfg       []byte `xorm:"TEXT"`
+#   Expected  []byte `xorm:"TEXT"`
+# }
+-
+  id: 1
+  type: 1
+  is_actived: false
+  cfg: "{\"Source\":{\"A\":\"string\",\"B\":1}}"
+  expected: "{\"Source\":{\"A\":\"string\",\"B\":1}}"
+-
+  id: 2
+  type: 2
+  is_actived: true
+  cfg: "{\"Source\":{\"A\":\"string2\",\"B\":2}}"
+  expected: "{\"A\":\"string2\",\"B\":2}"
+-
+  id: 3
+  type: 3
+  is_actived: false
+  cfg: "{\"Source\":{\"A\":\"string3\",\"B\":3}}"
+  expected: "{\"Source\":{\"A\":\"string3\",\"B\":3}}"
+-
+  id: 4
+  type: 4
+  is_actived: true
+  cfg: "{\"Source\":{\"A\":\"string4\",\"B\":4}}"
+  expected: "{\"Source\":{\"A\":\"string4\",\"B\":4}}"
+-
+  id: 5
+  type: 5
+  is_actived: false
+  cfg: "{\"Source\":{\"A\":\"string5\",\"B\":5}}"
+  expected: "{\"A\":\"string5\",\"B\":5}"
+-
+  id: 6
+  type: 2
+  is_actived: true
+  cfg: "{\"A\":\"string6\",\"B\":6}"
+  expected: "{\"A\":\"string6\",\"B\":6}"
+-
+  id: 7
+  type: 5
+  is_actived: false
+  cfg: "{\"A\":\"string7\",\"B\":7}"
+  expected: "{\"A\":\"string7\",\"B\":7}"

models/migrations/migrations.go

@@ -327,6 +327,8 @@ var migrations = []Migration{
 	NewMigration("Drop unneeded webhook related columns", dropWebhookColumns),
 	// v188 -> v189
 	NewMigration("Add key is verified to gpg key", addKeyIsVerified),
+	// v189 -> v190
+	NewMigration("Unwrap ldap.Sources", unwrapLDAPSourceCfg),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/migrations_test.go

@@ -220,6 +220,9 @@ func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En
 			if err := x.Close(); err != nil {
 				t.Errorf("error during close: %v", err)
 			}
+			if err := deleteDB(); err != nil {
+				t.Errorf("unable to reset database: %v", err)
+			}
 		}
 	}
 	if err != nil {

models/migrations/v189.go

@@ -0,0 +1,111 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	jsoniter "github.com/json-iterator/go"
+	"xorm.io/xorm"
+)
+
+func unwrapLDAPSourceCfg(x *xorm.Engine) error {
+	jsonUnmarshalHandleDoubleEncode := func(bs []byte, v interface{}) error {
+		json := jsoniter.ConfigCompatibleWithStandardLibrary
+		err := json.Unmarshal(bs, v)
+		if err != nil {
+			ok := true
+			rs := []byte{}
+			temp := make([]byte, 2)
+			for _, rn := range string(bs) {
+				if rn > 0xffff {
+					ok = false
+					break
+				}
+				binary.LittleEndian.PutUint16(temp, uint16(rn))
+				rs = append(rs, temp...)
+			}
+			if ok {
+				if rs[0] == 0xff && rs[1] == 0xfe {
+					rs = rs[2:]
+				}
+				err = json.Unmarshal(rs, v)
+			}
+		}
+		if err != nil && len(bs) > 2 && bs[0] == 0xff && bs[1] == 0xfe {
+			err = json.Unmarshal(bs[2:], v)
+		}
+		return err
+	}
+
+	// LoginSource represents an external way for authorizing users.
+	type LoginSource struct {
+		ID        int64 `xorm:"pk autoincr"`
+		Type      int
+		IsActived bool   `xorm:"INDEX NOT NULL DEFAULT false"`
+		IsActive  bool   `xorm:"INDEX NOT NULL DEFAULT false"`
+		Cfg       string `xorm:"TEXT"`
+	}
+
+	const ldapType = 2
+	const dldapType = 5
+
+	type WrappedSource struct {
+		Source map[string]interface{}
+	}
+
+	// change lower_email as unique
+	if err := x.Sync2(new(LoginSource)); err != nil {
+		return err
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+
+	const batchSize = 100
+	for start := 0; ; start += batchSize {
+		sources := make([]*LoginSource, 0, batchSize)
+		if err := sess.Limit(batchSize, start).Where("`type` = ? OR `type` = ?", ldapType, dldapType).Find(&sources); err != nil {
+			return err
+		}
+		if len(sources) == 0 {
+			break
+		}
+
+		for _, source := range sources {
+			wrapped := &WrappedSource{
+				Source: map[string]interface{}{},
+			}
+			err := jsonUnmarshalHandleDoubleEncode([]byte(source.Cfg), &wrapped)
+			if err != nil {
+				return fmt.Errorf("failed to unmarshal %s: %w", string(source.Cfg), err)
+			}
+			if wrapped.Source != nil && len(wrapped.Source) > 0 {
+				bs, err := jsoniter.Marshal(wrapped.Source)
+				if err != nil {
+					return err
+				}
+				source.Cfg = string(bs)
+				if _, err := sess.ID(source.ID).Cols("cfg").Update(source); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	if _, err := x.SetExpr("is_active", "is_actived").Update(&LoginSource{}); err != nil {
+		return fmt.Errorf("SetExpr Update failed:  %w", err)
+	}
+
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+	if err := dropTableColumns(sess, "login_source", "is_actived"); err != nil {
+		return err
+	}
+
+	return sess.Commit()
+}

models/migrations/v189_test.go

@@ -0,0 +1,83 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"testing"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/stretchr/testify/assert"
+)
+
+// LoginSource represents an external way for authorizing users.
+type LoginSourceOriginalV189 struct {
+	ID        int64 `xorm:"pk autoincr"`
+	Type      int
+	IsActived bool   `xorm:"INDEX NOT NULL DEFAULT false"`
+	Cfg       string `xorm:"TEXT"`
+	Expected  string `xorm:"TEXT"`
+}
+
+func (ls *LoginSourceOriginalV189) TableName() string {
+	return "login_source"
+}
+
+func Test_unwrapLDAPSourceCfg(t *testing.T) {
+
+	// Prepare and load the testing database
+	x, deferable := prepareTestEnv(t, 0, new(LoginSourceOriginalV189))
+	if x == nil || t.Failed() {
+		defer deferable()
+		return
+	}
+	defer deferable()
+
+	// LoginSource represents an external way for authorizing users.
+	type LoginSource struct {
+		ID       int64 `xorm:"pk autoincr"`
+		Type     int
+		IsActive bool   `xorm:"INDEX NOT NULL DEFAULT false"`
+		Cfg      string `xorm:"TEXT"`
+		Expected string `xorm:"TEXT"`
+	}
+
+	// Run the migration
+	if err := unwrapLDAPSourceCfg(x); err != nil {
+		assert.NoError(t, err)
+		return
+	}
+
+	const batchSize = 100
+	for start := 0; ; start += batchSize {
+		sources := make([]*LoginSource, 0, batchSize)
+		if err := x.Table("login_source").Limit(batchSize, start).Find(&sources); err != nil {
+			assert.NoError(t, err)
+			return
+		}
+
+		if len(sources) == 0 {
+			break
+		}
+
+		for _, source := range sources {
+			converted := map[string]interface{}{}
+			expected := map[string]interface{}{}
+
+			if err := jsoniter.Unmarshal([]byte(source.Cfg), &converted); err != nil {
+				assert.NoError(t, err)
+				return
+			}
+
+			if err := jsoniter.Unmarshal([]byte(source.Expected), &expected); err != nil {
+				assert.NoError(t, err)
+				return
+			}
+
+			assert.EqualValues(t, expected, converted, "unwrapLDAPSourceCfg failed for %d", source.ID)
+			assert.EqualValues(t, source.ID%2 == 0, source.IsActive, "unwrapLDAPSourceCfg failed for %d", source.ID)
+		}
+	}
+
+}