use SQLCipher (#981)

* use SQLCipher

* pass encryption key via CLI options

* update dependencies to use git

* add CONTRIBUTING.md

* move flag, enable build in sqlcipher branch

* update dependencies

.github/workflows/build.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - master
       - stable
+      - sqlcipher
     tags:
       - "v*"
   pull_request:
@@ -67,9 +68,9 @@ jobs:
       - name: Setup Stack
         uses: haskell/actions/setup@v1
         with:
-          ghc-version: '8.10.7'
+          ghc-version: "8.10.7"
           enable-stack: true
-          stack-version: 'latest'
+          stack-version: "latest"
 
       - name: Cache dependencies
         uses: actions/cache@v2

cabal.project

@@ -1,11 +1,26 @@
 packages: .
+-- packages: . ../simplexmq
+-- packages: . ../simplexmq ../direct-sqlcipher ../sqlcipher-simple
 
 constraints: zip +disable-bzip2 +disable-zstd
 
+package direct-sqlcipher
+    flags: +openssl
+
 source-repository-package
     type: git
     location: https://github.com/simplex-chat/simplexmq.git
-    tag: a7b39b710c3aab9b2a38bd6841e52e0342b3a7ef
+    tag: e4b77ed9e68373e2bad48a7c825db3860a6ad4d6
+
+source-repository-package
+    type: git
+    location: https://github.com/simplex-chat/direct-sqlcipher.git
+    tag: 477955063df65a2776c2a958b656ff359b76374d
+
+source-repository-package
+    type: git
+    location: https://github.com/simplex-chat/sqlcipher-simple.git
+    tag: 0738c7957e971b84a2a156d297596206b948c4f6
 
 source-repository-package
     type: git

docs/CONTRIBUTING.md

@@ -0,0 +1,16 @@
+# Contributing guide
+
+## Compiling with SQLCipher encryption enabled
+
+Add `cabal.project.local` to project root with the location of OpenSSL headers and libraries and flag setting encryption mode:
+
+```
+ignore-project: False
+
+package direct-sqlcipher
+    extra-include-dirs: /opt/homebrew/opt/openssl@3/include
+    extra-lib-dirs: /opt/homebrew/opt/openssl@3/lib
+    flags: +openssl
+```
+
+OpenSSL can be installed with `brew install openssl`

docs/rfcs/2022-08-29-database-encryption.md

@@ -0,0 +1,25 @@
+# Database encryption
+
+## Approach
+
+Using SQLCipher - it is a drop in replacement for SQLite that works for non-encrypted databases without any changes (TODO test on iOS/Android).
+
+`direct-sqlite` and `sqlite-simple` libraries are forked and renamed to `direct-sqlcipher` and `sqlcipher-simple`, with replaced cbits in `direct-sqlcipher` (TODO include SQLCipher as git submodule with a script to upgrade cbits).
+
+While SQLCipher provides additional C functions to set and change database key, they do not necessarily need to be exported as they are available as PRAGMAs.
+
+Moving from plaintext to encrypted database (and back) requires migration process using [sqlcipher_export() function](https://discuss.zetetic.net/t/how-to-encrypt-a-plaintext-sqlite-database-to-use-sqlcipher-and-avoid-file-is-encrypted-or-is-not-a-database-errors/868).
+
+The approach would be similar to database migration for the notifications:
+
+1. the current users will be offered to migrate to encrypted database once, with a notice that it can be done later via settings.
+2. the new users will be asked to enter a pass-phrase to create a new database (it can be empty, in which case the database won't be encrypted).
+3. during the migration the database backup will be created and the old database files will be preserved - in case of the app failing to open the new database right after the migration it should revert to using the previous database.
+
+When opening the database the key must be passed via chat command / agent configuration, some test query must be performed to check that the key is correct: https://www.zetetic.net/sqlcipher/sqlcipher-api/#PRAGMA_key
+
+Options to support in chat settings:
+
+- encrypt database (with automatic rollback in case of failure)
+- decrypt database (-"-)
+- change key (using [PRAGMA rekey](https://www.zetetic.net/sqlcipher/sqlcipher-api/#rekey))

package.yaml

@@ -35,7 +35,7 @@ dependencies:
   - simple-logger == 0.1.*
   - simplexmq >= 3.0
   - socks == 0.6.*
-  - sqlite-simple == 0.4.*
+  - sqlcipher-simple == 0.4.*
   - stm == 2.5.*
   - terminal == 0.2.*
   - text == 1.2.*

scripts/nix/sha256map.nix

@@ -1,5 +1,7 @@
 {
-  "https://github.com/simplex-chat/simplexmq.git"."a7b39b710c3aab9b2a38bd6841e52e0342b3a7ef" = "0iqk58dhckpij9l1z8bm83hghw5cwj9hmpkbk7j8vws123g1bd73";
+  "https://github.com/simplex-chat/simplexmq.git"."e4b77ed9e68373e2bad48a7c825db3860a6ad4d6" = "07p8g0a0pl61wrai2jyn311ys238s9kl1i98kpxsjifqif1h9wc1";
+  "https://github.com/simplex-chat/direct-sqlcipher.git"."477955063df65a2776c2a958b656ff359b76374d" = "1xiqid1344mwh3wnrczn6rxf59hml5g7kifah7skpd9javj4bb7s";
+  "https://github.com/simplex-chat/sqlcipher-simple.git"."0738c7957e971b84a2a156d297596206b948c4f6" = "0lysvzx2qzjcxka9w5cb0bnzym3nrqh7r7q5dw9h6g46vybc5lyc";
   "https://github.com/simplex-chat/aeson.git"."3eb66f9a68f103b5f1489382aad89f5712a64db7" = "0kilkx59fl6c3qy3kjczqvm8c3f4n3p0bdk9biyflf51ljnzp4yp";
   "https://github.com/simplex-chat/haskell-terminal.git"."f708b00009b54890172068f168bf98508ffcd495" = "0zmq7lmfsk8m340g47g5963yba7i88n4afa6z93sg9px5jv1mijj";
   "https://github.com/zw3rk/android-support.git"."3c3a5ab0b8b137a072c98d3d0937cbdc96918ddb" = "1r6jyxbim3dsvrmakqfyxbd6ms6miaghpbwyl0sr6dzwpgaprz97";

simplex-chat.cabal

@@ -90,7 +90,7 @@ library
     , simple-logger ==0.1.*
     , simplexmq >=3.0
     , socks ==0.6.*
-    , sqlite-simple ==0.4.*
+    , sqlcipher-simple ==0.4.*
     , stm ==2.5.*
     , terminal ==0.2.*
     , text ==1.2.*
@@ -132,7 +132,7 @@ executable simplex-bot
     , simplex-chat
     , simplexmq >=3.0
     , socks ==0.6.*
-    , sqlite-simple ==0.4.*
+    , sqlcipher-simple ==0.4.*
     , stm ==2.5.*
     , terminal ==0.2.*
     , text ==1.2.*
@@ -174,7 +174,7 @@ executable simplex-bot-advanced
     , simplex-chat
     , simplexmq >=3.0
     , socks ==0.6.*
-    , sqlite-simple ==0.4.*
+    , sqlcipher-simple ==0.4.*
     , stm ==2.5.*
     , terminal ==0.2.*
     , text ==1.2.*
@@ -217,7 +217,7 @@ executable simplex-chat
     , simplex-chat
     , simplexmq >=3.0
     , socks ==0.6.*
-    , sqlite-simple ==0.4.*
+    , sqlcipher-simple ==0.4.*
     , stm ==2.5.*
     , terminal ==0.2.*
     , text ==1.2.*
@@ -269,7 +269,7 @@ test-suite simplex-chat-test
     , simplex-chat
     , simplexmq >=3.0
     , socks ==0.6.*
-    , sqlite-simple ==0.4.*
+    , sqlcipher-simple ==0.4.*
     , stm ==2.5.*
     , terminal ==0.2.*
     , text ==1.2.*

src/Simplex/Chat.hs

@@ -86,6 +86,7 @@ defaultChatConfig =
         defaultAgentConfig
           { tcpPort = undefined, -- agent does not listen to TCP
             dbFile = "simplex_v1",
+            dbKey = "",
             yesToMigrations = False
           },
       yesToMigrations = False,
@@ -124,7 +125,7 @@ logCfg :: LogConfig
 logCfg = LogConfig {lc_file = Nothing, lc_stderr = True}
 
 newChatController :: SQLiteStore -> Maybe User -> ChatConfig -> ChatOpts -> Maybe (Notification -> IO ()) -> IO ChatController
-newChatController chatStore user cfg@ChatConfig {agentConfig = aCfg, tbqSize, defaultServers} ChatOpts {dbFilePrefix, smpServers, networkConfig, logConnections, logServerHosts} sendToast = do
+newChatController chatStore user cfg@ChatConfig {agentConfig = aCfg, tbqSize, defaultServers} ChatOpts {dbFilePrefix, dbKey, smpServers, networkConfig, logConnections, logServerHosts} sendToast = do
   let f = chatStoreFile dbFilePrefix
       config = cfg {subscriptionEvents = logConnections, hostEvents = logServerHosts}
       sendNotification = fromMaybe (const $ pure ()) sendToast
@@ -132,7 +133,7 @@ newChatController chatStore user cfg@ChatConfig {agentConfig = aCfg, tbqSize, de
   firstTime <- not <$> doesFileExist f
   currentUser <- newTVarIO user
   servers <- resolveServers defaultServers
-  smpAgent <- getSMPAgentClient aCfg {dbFile = dbFilePrefix <> "_agent.db"} servers {netCfg = networkConfig}
+  smpAgent <- getSMPAgentClient aCfg {dbFile = dbFilePrefix <> "_agent.db", dbKey} servers {netCfg = networkConfig}
   agentAsync <- newTVarIO Nothing
   idsDrg <- newTVarIO =<< drgNew
   inputQ <- newTBQueueIO tbqSize
@@ -1715,8 +1716,7 @@ processAgentMessage (Just user@User {userId, profile}) agentConnId agentMessage
       (probe, probeId) <- withStore $ \db -> createSentProbe db gVar userId ct
       void . sendDirectContactMessage ct $ XInfoProbe probe
       if connectedIncognito
-        then
+        then withStore' $ \db -> deleteSentProbe db userId probeId
-          withStore' $ \db -> deleteSentProbe db userId probeId
         else do
           cs <- withStore' $ \db -> getMatchingContacts db userId ct
           let probeHash = ProbeHash $ C.sha256Hash (unProbe probe)

src/Simplex/Chat/Core.hs

@@ -23,7 +23,7 @@ simplexChatCore cfg@ChatConfig {yesToMigrations} opts sendToast chat
   where
     initRun = do
       let f = chatStoreFile $ dbFilePrefix opts
-      st <- createStore f yesToMigrations
+      st <- createStore f (dbKey opts) yesToMigrations
       u <- getCreateActiveUser st
       cc <- newChatController st (Just u) cfg opts sendToast
       runSimplexChat opts u cc chat

src/Simplex/Chat/Mobile.hs

@@ -31,6 +31,8 @@ import System.Timeout (timeout)
 
 foreign export ccall "chat_init" cChatInit :: CString -> IO (StablePtr ChatController)
 
+foreign export ccall "chat_init_key" cChatInitKey :: CString -> CString -> IO (StablePtr ChatController)
+
 foreign export ccall "chat_send_cmd" cChatSendCmd :: StablePtr ChatController -> CString -> IO CJSONString
 
 foreign export ccall "chat_recv_msg" cChatRecvMsg :: StablePtr ChatController -> IO CJSONString
@@ -44,6 +46,12 @@ foreign export ccall "chat_parse_markdown" cChatParseMarkdown :: CString -> IO C
 cChatInit :: CString -> IO (StablePtr ChatController)
 cChatInit fp = peekCAString fp >>= chatInit >>= newStablePtr
 
+-- | initialize chat controller with encrypted database
+-- The active user has to be created and the chat has to be started before most commands can be used.
+cChatInitKey :: CString -> CString -> IO (StablePtr ChatController)
+cChatInitKey fp key =
+  ((,) <$> peekCAString fp <*> peekCAString key) >>= uncurry chatInitKey >>= newStablePtr
+
 -- | send command to chat (same syntax as in terminal for now)
 cChatSendCmd :: StablePtr ChatController -> CString -> IO CJSONString
 cChatSendCmd cPtr cCmd = do
@@ -67,6 +75,7 @@ mobileChatOpts :: ChatOpts
 mobileChatOpts =
   ChatOpts
     { dbFilePrefix = undefined,
+      dbKey = "",
       smpServers = [],
       networkConfig = defaultNetworkConfig,
       logConnections = False,
@@ -91,9 +100,12 @@ getActiveUser_ :: SQLiteStore -> IO (Maybe User)
 getActiveUser_ st = find activeUser <$> withTransaction st getUsers
 
 chatInit :: String -> IO ChatController
-chatInit dbFilePrefix = do
+chatInit = (`chatInitKey` "")
+
+chatInitKey :: String -> String -> IO ChatController
+chatInitKey dbFilePrefix dbKey = do
   let f = chatStoreFile dbFilePrefix
-  chatStore <- createStore f (yesToMigrations (defaultMobileConfig :: ChatConfig))
+  chatStore <- createStore f dbKey (yesToMigrations (defaultMobileConfig :: ChatConfig))
   user_ <- getActiveUser_ chatStore
   newChatController chatStore user_ defaultMobileConfig mobileChatOpts {dbFilePrefix} Nothing
 

src/Simplex/Chat/Options.hs

@@ -25,6 +25,7 @@ import System.FilePath (combine)
 
 data ChatOpts = ChatOpts
   { dbFilePrefix :: String,
+    dbKey :: String,
     smpServers :: [SMPServer],
     networkConfig :: NetworkConfig,
     logConnections :: Bool,
@@ -47,6 +48,14 @@ chatOpts appDir defaultDbFileName = do
           <> value defaultDbFilePath
           <> showDefault
       )
+  dbKey <-
+    strOption
+      ( long "key"
+          <> short 'k'
+          <> metavar "KEY"
+          <> help "Database encryption key/pass-phrase"
+          <> value ""
+      )
   smpServers <-
     option
       parseSMPServers
@@ -126,6 +135,7 @@ chatOpts appDir defaultDbFileName = do
   pure
     ChatOpts
       { dbFilePrefix,
+        dbKey,
         smpServers,
         networkConfig = fullNetworkConfig socksProxy $ useTcpTimeout socksProxy t,
         logConnections,

src/Simplex/Chat/Store.hs

@@ -276,8 +276,8 @@ migrations = sortBy (compare `on` name) $ map migration schemaMigrations
   where
     migration (name, query) = Migration {name = name, up = fromQuery query}
 
-createStore :: FilePath -> Bool -> IO SQLiteStore
+createStore :: FilePath -> String -> Bool -> IO SQLiteStore
-createStore dbFilePath = createSQLiteStore dbFilePath migrations
+createStore dbFilePath dbKey = createSQLiteStore dbFilePath dbKey migrations
 
 chatStoreFile :: FilePath -> FilePath
 chatStoreFile = (<> "_chat.db")

stack.yaml

@@ -49,7 +49,13 @@ extra-deps:
   # - simplexmq-1.0.0@sha256:34b2004728ae396e3ae449cd090ba7410781e2b3cefc59259915f4ca5daa9ea8,8561
   # - ../simplexmq
   - github: simplex-chat/simplexmq
-    commit: a7b39b710c3aab9b2a38bd6841e52e0342b3a7ef
+    commit: e4b77ed9e68373e2bad48a7c825db3860a6ad4d6
+  # - ../direct-sqlcipher
+  - github: simplex-chat/direct-sqlcipher
+    commit: 477955063df65a2776c2a958b656ff359b76374d
+  # - ../sqlcipher-simple
+  - github: simplex-chat/sqlcipher-simple
+    commit: 0738c7957e971b84a2a156d297596206b948c4f6
   # - terminal-0.2.0.0@sha256:de6770ecaae3197c66ac1f0db5a80cf5a5b1d3b64a66a05b50f442de5ad39570,2977
   - github: simplex-chat/aeson
     commit: 3eb66f9a68f103b5f1489382aad89f5712a64db7

tests/ChatClient.hs

@@ -48,6 +48,8 @@ testOpts :: ChatOpts
 testOpts =
   ChatOpts
     { dbFilePrefix = undefined,
+      dbKey = "",
+      -- dbKey = "this is a pass-phrase to encrypt the database",
       smpServers = ["smp://LcJUMfVhwD8yxjAiSaDzzGF3-kLG4Uh0Fl_ZIjrRwjI=@localhost:5001"],
       networkConfig = defaultNetworkConfig,
       logConnections = False,
@@ -101,16 +103,16 @@ testCfgV1 :: ChatConfig
 testCfgV1 = testCfg {agentConfig = testAgentCfgV1}
 
 createTestChat :: ChatConfig -> ChatOpts -> String -> Profile -> IO TestCC
-createTestChat cfg opts dbPrefix profile = do
+createTestChat cfg opts@ChatOpts {dbKey} dbPrefix profile = do
   let dbFilePrefix = testDBPrefix <> dbPrefix
-  st <- createStore (dbFilePrefix <> "_chat.db") False
+  st <- createStore (dbFilePrefix <> "_chat.db") dbKey False
   Right user <- withTransaction st $ \db -> runExceptT $ createUser db profile True
   startTestChat_ st cfg opts dbFilePrefix user
 
 startTestChat :: ChatConfig -> ChatOpts -> String -> IO TestCC
-startTestChat cfg opts dbPrefix = do
+startTestChat cfg opts@ChatOpts {dbKey} dbPrefix = do
   let dbFilePrefix = testDBPrefix <> dbPrefix
-  st <- createStore (dbFilePrefix <> "_chat.db") False
+  st <- createStore (dbFilePrefix <> "_chat.db") dbKey False
   Just user <- find activeUser <$> withTransaction st getUsers
   startTestChat_ st cfg opts dbFilePrefix user
 

tests/MobileTests.hs

@@ -82,7 +82,7 @@ testChatApiNoUser = withTmpFiles $ do
 testChatApi :: IO ()
 testChatApi = withTmpFiles $ do
   let f = chatStoreFile $ testDBPrefix <> "1"
-  st <- createStore f True
+  st <- createStore f "" True
   Right _ <- withTransaction st $ \db -> runExceptT $ createUser db aliceProfile True
   cc <- chatInit $ testDBPrefix <> "1"
   chatSendCmd cc "/u" `shouldReturn` activeUser

tests/SchemaDump.hs

@@ -22,7 +22,7 @@ schemaDumpTest =
 testVerifySchemaDump :: IO ()
 testVerifySchemaDump =
   withTmpFiles $ do
-    void $ createStore testDB False
+    void $ createStore testDB "" False
     void $ readCreateProcess (shell $ "touch " <> schema) ""
     savedSchema <- readFile schema
     savedSchema `deepseq` pure ()