Merge branch 'master' into ep/v1-rc

apps/smp-server/Main.hs

@@ -181,10 +181,10 @@ initializeServer InitOptions {enableStoreLog, signAlgorithm} = do
         \# and restoring them when the server is started.\n\
         \# Log is compacted on start (deleted queues are removed).\n\
         \# The messages in the queues are not logged.\n"
-          <> ("enable: " <> (if enableStoreLog then "on" else "off # on") <> "\n\n")
+          <> ("enable: " <> (if enableStoreLog then "on" else "off  # on") <> "\n\n")
           <> "[TRANSPORT]\n\
              \port: 5223\n\
-             \websockets: on\n"
+             \websockets: off\n"
 
     warnCAPrivateKeyFile =
       putStrLn $

scripts/smp-server-digitalocean-droplet/files/opt/simplex/on_login.sh

@@ -1,17 +1,17 @@
 #!/bin/bash
-# receives fingerprint file location as the first parameter
+
+fingerprint=$1
 
 ip_address=$(hostname -I | awk '{print$1}')
-hash=$(cat $1)
 
 cat <<EOF
 ********************************************************************************
 
-SMP server address: $ip_address#$hash
+SMP server address: $ip_address#$fingerprint
 Check SMP server status with: systemctl status smp-server
 
 To keep this server secure, the UFW firewall is enabled.
-All ports are BLOCKED except 22 (SSH), 80 (HTTP), 5223 (SMP server).
+All ports are BLOCKED except 22 (SSH), 443 (HTTPS), 5223 (SMP server).
 
 ********************************************************************************
 To stop seeing this message delete line - bash /opt/simplex/on_login.sh - from /root/.bashrc

scripts/smp-server-digitalocean-droplet/scripts/01-packages.sh

@@ -1,8 +1,25 @@
 #!/bin/sh
 
-# update apt
-apt-get -y update
-apt-get -y upgrade
+# https://superuser.com/questions/1638779/automatic-yess-to-linux-update-upgrade
+# https://superuser.com/questions/1412054/non-interactive-apt-upgrade
+sudo DEBIAN_FRONTEND=noninteractive \
+  apt-get \
+  -o Dpkg::Options::=--force-confold \
+  -o Dpkg::Options::=--force-confdef \
+  -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \
+  update
 
-# packages needed for smp-server init-start.sh
-apt-get install -y jq
+sudo DEBIAN_FRONTEND=noninteractive \
+  apt-get \
+  -o Dpkg::Options::=--force-confold \
+  -o Dpkg::Options::=--force-confdef \
+  -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \
+  dist-upgrade
+
+# TODO install unattended-upgrades; jq is not needed on DigitalOcean
+# sudo DEBIAN_FRONTEND=noninteractive \
+#   apt-get \
+#   -o Dpkg::Options::=--force-confold \
+#   -o Dpkg::Options::=--force-confdef \
+#   -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \
+#   install jq

scripts/smp-server-digitalocean-droplet/scripts/02-firewall.sh

@@ -1,9 +1,9 @@
 #!/bin/sh
 
-# add firewall
+# Add firewall
 echo "y" | ufw enable
 
-# open ports
+# Open ports
 ufw allow ssh
-ufw allow http
+ufw allow https
 ufw allow 5223

scripts/smp-server-digitalocean-droplet/scripts/03-init-start.sh

@@ -1,57 +1,50 @@
 #!/bin/bash
 
+# Download latest release
 bin_dir="/opt/simplex/bin"
-conf_dir="/etc/opt/simplex"
-
+binary="$bin_dir/smp-server"
 mkdir -p $bin_dir
+curl -L -o $binary https://github.com/simplex-chat/simplexmq/releases/latest/download/smp-server-ubuntu-20_04-x86-64
+chmod +x $binary
 
-echo "downloading the latest SMP server release"
-curl -s https://api.github.com/repos/simplex-chat/simplexmq/releases/latest > release.json
-jq '.assets[].browser_download_url | select(test("smp-server-ubuntu-20_04-x86-64"))' release.json \
-| tr -d \" \
-| wget -qi -
-
-release_version=$(jq '.tag_name' release.json | tr -d \")
-echo "downloaded SMP server $release_version"
-rm release.json
-
-echo "preparing for SMP server initiaization"
-mv smp-server-ubuntu-20_04-x86-64 $bin_dir/smp-server
-chmod +x $bin_dir/smp-server
-
+# / Add to PATH
 cat <<EOT >> /etc/profile.d/simplex.sh
 #!/bin/bash
 
 export PATH="$PATH:$bin_dir"
 
 EOT
-source /etc/profile.d/simplex.sh
+# Add to PATH /
 
-# prepare SMP server systemd service
+# Source and test PATH
+source /etc/profile.d/simplex.sh
+smp-server --version
+
+# Initialize server
+smp-server init -l
+
+# Server fingerprint
+fingerprint=$(cat /etc/opt/simplex/fingerprint)
+
+# On login script
+echo "bash /opt/simplex/on_login.sh $fingerprint" >> /root/.bashrc
+
+# / Create systemd service
 cat <<EOT >> /etc/systemd/system/smp-server.service
 [Unit]
 Description=SMP server systemd service
 
 [Service]
 Type=simple
-ExecStart=/bin/sh -c "$bin_dir/smp-server start"
+ExecStart=/bin/sh -c "$binary start"
 
 [Install]
 WantedBy=multi-user.target
 
 EOT
+# Create systemd service /
+
+# Start systemd service
 chmod 644 /etc/systemd/system/smp-server.service
-
-echo "initializing SMP server"
-smp-server init -l
-# CA certificate (identity/offline) fingerprint
-hash_file="$conf_dir/fingerprint"
-# turn off websockets support
-sed -e '/websockets/s/^/# /g' -i $conf_dir/smp-server.ini
-
-# add welcome script to .bashrc
-echo "bash /opt/simplex/on_login.sh $hash_file" >> /root/.bashrc
-
-echo "starting SMP server"
 sudo systemctl enable smp-server
 sudo systemctl start smp-server

scripts/smp-server-linode.sh

@@ -1,125 +1,148 @@
 #!/bin/bash
+
 # <UDF name="enable_store_log" label="Store log - persists SMP queues to append only log and restores them upon server restart." default="on" oneof="on, off" />
 # <UDF name="api_token" label="Linode API token - enables StackScript to create tags containing SMP server FQDN / IP address, CA certificate fingerprint and server version. Use `fqdn#fingerprint` or `ip#fingerprint` as SMP server address in the client. Note: minimal permissions token should have are - read/write access to `linodes` (to update linode tags) and `domains` (to add A record for the chosen 3rd level domain)" default="" />
 # <UDF name="fqdn" label="FQDN (Fully qualified domain name) - provide third level domain name (ex: smp.example.com). If provided can be used instead of IP address." default="" />
 
-# log all stdout output to stackscript.log
+# Log all stdout output to stackscript.log
 exec &> >(tee -i /var/log/stackscript.log)
-# uncomment next line to enable debugging features
+
+# Uncomment next line to enable debugging features
 # set -xeo pipefail
 
 cd $HOME
 
-sudo apt-get -y update 
-sudo apt-get -y upgrade
-sudo apt-get install -y jq
+# https://superuser.com/questions/1638779/automatic-yess-to-linux-update-upgrade
+# https://superuser.com/questions/1412054/non-interactive-apt-upgrade
+sudo DEBIAN_FRONTEND=noninteractive \
+  apt-get \
+  -o Dpkg::Options::=--force-confold \
+  -o Dpkg::Options::=--force-confdef \
+  -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \
+  update
 
-# add firewall
+sudo DEBIAN_FRONTEND=noninteractive \
+  apt-get \
+  -o Dpkg::Options::=--force-confold \
+  -o Dpkg::Options::=--force-confdef \
+  -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \
+  dist-upgrade
+
+# TODO install unattended-upgrades
+sudo DEBIAN_FRONTEND=noninteractive \
+  apt-get \
+  -o Dpkg::Options::=--force-confold \
+  -o Dpkg::Options::=--force-confdef \
+  -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \
+  install jq
+
+# Add firewall
 echo "y" | ufw enable
-# open ports
+
+# Open ports
 ufw allow ssh
-ufw allow http
+ufw allow https
 ufw allow 5223
 
+# Download latest release
 bin_dir="/opt/simplex/bin"
-conf_dir="/etc/opt/simplex"
-
+binary="$bin_dir/smp-server"
 mkdir -p $bin_dir
+curl -L -o $binary https://github.com/simplex-chat/simplexmq/releases/latest/download/smp-server-ubuntu-20_04-x86-64
+chmod +x $binary
 
-# retrieve latest release info and download smp-server executable
-curl -s https://api.github.com/repos/simplex-chat/simplexmq/releases/latest > release.json
-jq '.assets[].browser_download_url | select(test("smp-server-ubuntu-20_04-x86-64"))' release.json \
-| tr -d \" \
-| wget -qi -
-
-mv smp-server-ubuntu-20_04-x86-64 $bin_dir/smp-server
-chmod +x $bin_dir/smp-server
-
+# / Add to PATH
 cat <<EOT >> /etc/profile.d/simplex.sh
 #!/bin/bash
 
 export PATH="$PATH:$bin_dir"
 
 EOT
-source /etc/profile.d/simplex.sh
+# Add to PATH /
 
-# initialize SMP server
+# Source and test PATH
+source /etc/profile.d/simplex.sh
+smp-server --version
+
+# Initialize server
 init_opts=()
 [[ $ENABLE_STORE_LOG == "on" ]] && init_opts+=(-l)
 smp-server init "${init_opts[@]}"
-# CA certificate (identity/offline) fingerprint
-hash_file="$conf_dir/fingerprint"
-# turn off websockets support
-sed -e '/websockets/s/^/# /g' -i $conf_dir/smp-server.ini
 
-# create script that will run on login
+# Server fingerprint
+fingerprint=$(cat /etc/opt/simplex/fingerprint)
+
+# On login script
 on_login_script="/opt/simplex/on_login.sh"
+
 cat <<EOT >> $on_login_script
 #!/bin/bash
-# receives fingerprint file location as the first parameter
+
+fingerprint=\$1
 
 ip_address=\$(hostname -I | awk '{print\$1}')
-hash=\$(cat \$1)
 
 cat <<EOF
 ********************************************************************************
 
-SMP server address: \$ip_address#\$hash
+SMP server address: \$ip_address#\$fingerprint
 Check SMP server status with: systemctl status smp-server
 
 To keep this server secure, the UFW firewall is enabled.
-All ports are BLOCKED except 22 (SSH), 80 (HTTP), 5223 (SMP server).
+All ports are BLOCKED except 22 (SSH), 443 (HTTPS), 5223 (SMP server).
 
 ********************************************************************************
 To stop seeing this message delete line - bash /opt/simplex/on_login.sh - from /root/.bashrc
 EOF
 
 EOT
+
 chmod +x $on_login_script
-echo "bash $on_login_script $hash_file" >> /root/.bashrc
+echo "bash $on_login_script $fingerprint" >> /root/.bashrc
 
-# create A record and update linode's tags
+# Create A record and update Linode's tags
 if [ ! -z "$API_TOKEN" ]; then
-     ip_address=$(curl ifconfig.me)
-     address=$ip_address
-     if [ ! -z "$FQDN" ]; then
-          domain_address=$(echo $FQDN | rev | cut -d "." -f 1,2 | rev)
-          domain_id=$(curl -H "Authorization: Bearer $API_TOKEN" https://api.linode.com/v4/domains \
-          | jq --arg da "$domain_address" '.data[] | select( .domain == $da ) | .id')
-          if [[ ! -z $domain_id ]]; then
-               curl -s -H "Content-Type: application/json" \
-                    -H "Authorization: Bearer $API_TOKEN" \
-                    -X POST -d "{\"type\":\"A\",\"name\":\"$FQDN\",\"target\":\"$ip_address\"}" \
-                    https://api.linode.com/v4/domains/${domain_id}/records
-               address=$FQDN
-          fi
-     fi
+  ip_address=$(curl ifconfig.me)
+  address=$ip_address
+  if [ ! -z "$FQDN" ]; then
+    domain_address=$(echo $FQDN | rev | cut -d "." -f 1,2 | rev)
+    domain_id=$(curl -H "Authorization: Bearer $API_TOKEN" https://api.linode.com/v4/domains \
+    | jq --arg da "$domain_address" '.data[] | select( .domain == $da ) | .id')
+    if [[ ! -z $domain_id ]]; then
+      curl \
+        -s -H "Content-Type: application/json" \
+        -H "Authorization: Bearer $API_TOKEN" \
+        -X POST -d "{\"type\":\"A\",\"name\":\"$FQDN\",\"target\":\"$ip_address\"}" \
+        https://api.linode.com/v4/domains/${domain_id}/records
+      address=$FQDN
+    fi
+  fi
 
-     hash=$(cat $hash_file)
-     release_version=$(jq '.tag_name' release.json | tr -d \")
+  version=$(smp-server --version | cut -d ' ' -f 3-)
 
-     curl -s -H "Content-Type: application/json" \
-          -H "Authorization: Bearer $API_TOKEN" \
-          -X PUT -d "{\"tags\":[\"$address\",\"#$hash\",\"$release_version\"]}" \
-          https://api.linode.com/v4/linode/instances/$LINODE_ID
+  curl \
+    -s -H "Content-Type: application/json" \
+    -H "Authorization: Bearer $API_TOKEN" \
+    -X PUT -d "{\"tags\":[\"$address\",\"#$fingerprint\",\"$version\"]}" \
+    https://api.linode.com/v4/linode/instances/$LINODE_ID
 fi
 
-# create, enable and start SMP server systemd service
+# / Create systemd service
 cat <<EOT >> /etc/systemd/system/smp-server.service
 [Unit]
 Description=SMP server systemd service
 
 [Service]
 Type=simple
-ExecStart=/bin/sh -c "$bin_dir/smp-server start"
+ExecStart=/bin/sh -c "$binary start"
 
 [Install]
 WantedBy=multi-user.target
 
 EOT
+# Create systemd service /
+
+# Start systemd service
 chmod 644 /etc/systemd/system/smp-server.service
 sudo systemctl enable smp-server
 sudo systemctl start smp-server
-
-# cleanup
-rm release.json

simplexmq.cabal

@@ -34,7 +34,6 @@ library
       Simplex.Messaging.Agent
       Simplex.Messaging.Agent.Client
       Simplex.Messaging.Agent.Env.SQLite
-      Simplex.Messaging.Agent.ExceptT
       Simplex.Messaging.Agent.Protocol
       Simplex.Messaging.Agent.QueryString
       Simplex.Messaging.Agent.RetryInterval

src/Simplex/Messaging/Agent/Client.hs

@@ -2,11 +2,14 @@
 {-# LANGUAGE DataKinds #-}
 {-# LANGUAGE DuplicateRecordFields #-}
 {-# LANGUAGE FlexibleContexts #-}
+{-# LANGUAGE InstanceSigs #-}
 {-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE NamedFieldPuns #-}
 {-# LANGUAGE OverloadedLists #-}
 {-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE RankNTypes #-}
 {-# LANGUAGE ScopedTypeVariables #-}
+{-# OPTIONS_GHC -fno-warn-orphans #-}
 
 module Simplex.Messaging.Agent.Client
   ( AgentClient (..),
@@ -50,7 +53,6 @@ import Data.Set (Set)
 import qualified Data.Set as S
 import Data.Text.Encoding
 import Simplex.Messaging.Agent.Env.SQLite
-import Simplex.Messaging.Agent.ExceptT ()
 import Simplex.Messaging.Agent.Protocol
 import Simplex.Messaging.Agent.RetryInterval
 import Simplex.Messaging.Agent.Store
@@ -61,7 +63,7 @@ import Simplex.Messaging.Protocol (QueueId, QueueIdsKeys (..), SndPublicVerifyKe
 import qualified Simplex.Messaging.Protocol as SMP
 import Simplex.Messaging.Util (bshow, liftEitherError, liftError)
 import Simplex.Messaging.Version
-import UnliftIO.Exception (IOException)
+import UnliftIO.Exception (Exception, IOException)
 import qualified UnliftIO.Exception as E
 import UnliftIO.STM
 
@@ -102,6 +104,18 @@ newAgentClient agentEnv = do
 -- | Agent monad with MonadReader Env and MonadError AgentErrorType
 type AgentMonad m = (MonadUnliftIO m, MonadReader Env m, MonadError AgentErrorType m)
 
+newtype InternalException e = InternalException {unInternalException :: e}
+  deriving (Eq, Show)
+
+instance Exception e => Exception (InternalException e)
+
+instance (MonadUnliftIO m, Exception e) => MonadUnliftIO (ExceptT e m) where
+  withRunInIO :: ((forall a. ExceptT e m a -> IO a) -> IO b) -> ExceptT e m b
+  withRunInIO exceptToIO =
+    withExceptT unInternalException . ExceptT . E.try $
+      withRunInIO $ \run ->
+        exceptToIO $ run . (either (E.throwIO . InternalException) return <=< runExceptT)
+
 getSMPServerClient :: forall m. AgentMonad m => AgentClient -> SMPServer -> m SMPClient
 getSMPServerClient c@AgentClient {smpClients, msgQ} srv =
   readTVarIO smpClients

src/Simplex/Messaging/Agent/ExceptT.hs

@@ -1,22 +0,0 @@
-{-# LANGUAGE InstanceSigs #-}
-{-# LANGUAGE RankNTypes #-}
-{-# OPTIONS_GHC -fno-warn-orphans #-}
-
-module Simplex.Messaging.Agent.ExceptT where
-
-import Control.Monad.Except
-import Control.Monad.IO.Unlift
-import UnliftIO.Exception (Exception)
-import qualified UnliftIO.Exception as E
-
-newtype InternalException e = InternalException {unInternalException :: e}
-  deriving (Eq, Show)
-
-instance Exception e => Exception (InternalException e)
-
-instance (MonadUnliftIO m, Exception e) => MonadUnliftIO (ExceptT e m) where
-  withRunInIO :: ((forall a. ExceptT e m a -> IO a) -> IO b) -> ExceptT e m b
-  withRunInIO exceptToIO =
-    withExceptT unInternalException . ExceptT . E.try $
-      withRunInIO $ \run ->
-        exceptToIO $ run . (either (E.throwIO . InternalException) return <=< runExceptT)

src/Simplex/Messaging/Agent/Protocol.hs

@@ -107,6 +107,7 @@ where
 
 import Control.Applicative (optional, (<|>))
 import Control.Monad.IO.Class
+import Data.Aeson (FromJSON (..), ToJSON (..))
 import Data.Attoparsec.ByteString.Char8 (Parser)
 import qualified Data.Attoparsec.ByteString.Char8 as A
 import Data.ByteString.Base64
@@ -438,6 +439,20 @@ instance StrEncoding AConnectionRequestUri where
     where
       crModeP = "invitation" $> CMInvitation <|> "contact" $> CMContact
 
+instance ConnectionModeI m => FromJSON (ConnectionRequestUri m) where
+  parseJSON = strParseJSON "ConnectionRequestUri"
+
+instance ConnectionModeI m => ToJSON (ConnectionRequestUri m) where
+  toJSON = strToJSON
+  toEncoding = strToJEncoding
+
+instance FromJSON AConnectionRequestUri where
+  parseJSON = strParseJSON "ConnectionRequestUri"
+
+instance ToJSON AConnectionRequestUri where
+  toJSON = strToJSON
+  toEncoding = strToJEncoding
+
 -- debug :: Show a => String -> a -> a
 -- debug name value = unsafePerformIO (putStrLn $ name <> ": " <> show value) `seq` value
 -- {-# INLINE debug #-}

tests/AgentTests/SQLiteTests.hs

@@ -21,7 +21,7 @@ import Data.Word (Word32)
 import qualified Database.SQLite.Simple as DB
 import Database.SQLite.Simple.QQ (sql)
 import SMPClient (testKeyHash)
-import Simplex.Messaging.Agent.ExceptT ()
+import Simplex.Messaging.Agent.Client ()
 import Simplex.Messaging.Agent.Protocol
 import Simplex.Messaging.Agent.Store
 import Simplex.Messaging.Agent.Store.SQLite