smp server: deliver messages sent while the recipient subscribes (#2008)

* tests: reproduce SUB and SEND race in one block

* smp server: move subscriber registration out of serverThread

* smp server: deliver messages sent while the recipient subscribes

* smp server: keep subscriber registration in smpServer, pass it to client

* tests: receive SUB and SEND responses in order

* smp server: register subscription of a new queue directly

* smp server: name server subscribers srvSubs in runClient

* tests: sign SUB with authTransmission, keep test helpers unchanged

* smp server: remove do and mask_ from subscribeNewQueue

* smp server: send the first message once when SEND delivers it during SUB

* smp server: plan with SUB and SEND sequences before and after the change

* smp server: one sequence diagram with added and removed steps in the plan

* smp server: process all subscription changes in the client thread, remove subQ and serverThread

* smp server: send the first message once after SUBS, keep service counts exact

* smp server: name the reader monad in the type of setService

* plans: correct the diagrams and decisions of subscription processing

* plans: diagrams of a duplicate SUB and of SUBS, corrected decisions

* plans: duplicate SUB with SEND, service counts before and after the change

* tests: switch subscription with SUB and SEND in one block

* docs: remove the outdated SMP server design diagram

* smp server: move processClientSub back to the place of serverThread, passed to client threads as a parameter

* smp server: pass the pair of subscription processors to client, applied to the client where called

* smp server: show subscribeQueue and sharedSubscribeQueue as comments at their master places, temporary [skip ci]

* smp server: remove the temporary comments, the tree equals 966701dd [skip ci]

* tests: cover the changed subscription paths with concurrent, blocked, service and notifier scenarios

* tests: cover SMP server subscription changes

* smp server: remove the subscription branch for a disconnected client
This commit is contained in:
Evgeny @ SimpleX Chat authored and GitHub committed 2026-11-09 12:41:55 +00:00
1 parent 402548eb55
commit 71bb0d34a4
7 files changed
+503 -512

No files matched your search

-4
View File
@@ -308,10 +308,6 @@ smp-server delete
smp-server init [-l] -n <fqdn>
```
## SMP server design
![SMP server design](./design/server.svg)
## SMP agent design
![SMP agent design](./design/agent2.svg)
-57
View File
@@ -1,57 +0,0 @@
digraph SMPServer {
graph [fontname=arial]
node [fontname=arial fontsize=11 shape=box]
edge [fontname=arial fontsize=10 arrowhead=open arrowtail=open]
subgraph clusterPersistence {
label="persistence (STM)"
msgQueues [shape=cylinder label="Message\nqueues"]
queueStore [shape=cylinder label="SMP queue\nrecords"]
}
subgraph clusterServer {
label="server threads"
main [shape=hexagon color=orange label="main\nthread"]
ss [label="server TCP socket" color=blue]
subgraph clusterThreads {
label=""
node [shape=hexagon color=orange]
runClient [label="runClient\nthread"]
server [label="server\nthread"]
}
subscribedQ [shape="larrow" label="subscribed\nTBQueue"]
main -> {server runClient} [style=dashed label=race color=orange fontcolor=orange]
ss -> runClient [color=blue]
subscribedQ -> server
}
subgraph clusterConnection {
label="1 group per client connection"
cs [label="client connection TCP socket" color=blue]
subgraph clusterThreads {
node [shape=hexagon, color=orange]
label=""
receive [label="receive\nthread"]
client [label="client\nthread"]
send [label="send\nthread"]
}
runClient -> cs [style=dashed label="connect" color=blue fontcolor=blue]
runClient -> {client receive send} [style=dashed label=race color=orange fontcolor=orange]
server -> inq [label="END"]
subscriber [shape=hexagon color=orange label="subscriber\nthread"]
inq [shape=rarrow label="receive\nTBQueue"]
outq [shape=larrow label="send\nTBQueue"]
cs -> receive -> inq -> client [color=blue]
msgQueues -> subscriber [label="Message" color=green fontcolor=green]
subscriber -> outq [label="MSG" color=blue fontcolor=blue constraint=false]
client -> queueStore [dir=both]
client -> subscriber [style=dashed label="1 fork per\nSMP queue" color=orange]
client -> msgQueues [dir="both" label="SEND,\nSUB,\nACK" color=green]
client -> outq -> send -> cs [color=blue]
client -> subscribedQ [label="(rId, Client)"]
}
}
-277
View File
@@ -1,277 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Generated by graphviz version 2.40.1 (20161225.0304)
-->
<!-- Title: SMPServer Pages: 1 -->
<svg width="855pt" height="677pt"
viewBox="0.00 0.00 855.00 677.40" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 673.3999)">
<title>SMPServer</title>
<polygon fill="#ffffff" stroke="transparent" points="-4,4 -4,-673.3999 851,-673.3999 851,4 -4,4"/>
<g id="clust1" class="cluster">
<title>clusterPersistence</title>
<polygon fill="none" stroke="#000000" points="671,-112.175 671,-200.275 839,-200.275 839,-112.175 671,-112.175"/>
<text text-anchor="middle" x="755" y="-183.675" font-family="arial" font-size="14.00" fill="#000000">persistence (STM)</text>
</g>
<g id="clust2" class="cluster">
<title>clusterServer</title>
<polygon fill="none" stroke="#000000" points="8,-356.4499 8,-661.3999 314,-661.3999 314,-356.4499 8,-356.4499"/>
<text text-anchor="middle" x="161" y="-644.7999" font-family="arial" font-size="14.00" fill="#000000">server threads</text>
</g>
<g id="clust3" class="cluster">
<title>clusterThreads</title>
<polygon fill="none" stroke="#000000" points="96,-364.4499 96,-436.6249 306,-436.6249 306,-364.4499 96,-364.4499"/>
</g>
<g id="clust5" class="cluster">
<title>clusterConnection</title>
<polygon fill="none" stroke="#000000" points="322,-8 322,-545.4249 663,-545.4249 663,-8 322,-8"/>
<text text-anchor="middle" x="492.5" y="-528.8249" font-family="arial" font-size="14.00" fill="#000000">1 group per client connection</text>
</g>
<g id="clust6" class="cluster">
<title>clusterThreads</title>
<polygon fill="none" stroke="#000000" points="330,-254.275 330,-326.4499 621,-326.4499 621,-254.275 330,-254.275"/>
</g>
<!-- msgQueues -->
<g id="node1" class="node">
<title>msgQueues</title>
<path fill="none" stroke="#000000" d="M831.1227,-163.2985C831.1227,-165.6863 817.6212,-167.6259 801,-167.6259 784.3788,-167.6259 770.8773,-165.6863 770.8773,-163.2985 770.8773,-163.2985 770.8773,-124.3515 770.8773,-124.3515 770.8773,-121.9636 784.3788,-120.024 801,-120.024 817.6212,-120.024 831.1227,-121.9636 831.1227,-124.3515 831.1227,-124.3515 831.1227,-163.2985 831.1227,-163.2985"/>
<path fill="none" stroke="#000000" d="M831.1227,-163.2985C831.1227,-160.9107 817.6212,-158.971 801,-158.971 784.3788,-158.971 770.8773,-160.9107 770.8773,-163.2985"/>
<text text-anchor="middle" x="801" y="-147.125" font-family="arial" font-size="11.00" fill="#000000">Message</text>
<text text-anchor="middle" x="801" y="-133.925" font-family="arial" font-size="11.00" fill="#000000">queues</text>
</g>
<!-- subscriber -->
<g id="node13" class="node">
<title>subscriber</title>
<polygon fill="none" stroke="#ffa500" points="655.4603,-44.0875 628.2302,-72.2627 573.7698,-72.2627 546.5397,-44.0875 573.7698,-15.9122 628.2302,-15.9122 655.4603,-44.0875"/>
<text text-anchor="middle" x="601" y="-47.3875" font-family="arial" font-size="11.00" fill="#000000">subscriber</text>
<text text-anchor="middle" x="601" y="-34.1875" font-family="arial" font-size="11.00" fill="#000000">thread</text>
</g>
<!-- msgQueues&#45;&gt;subscriber -->
<g id="edge13" class="edge">
<title>msgQueues&#45;&gt;subscriber</title>
<path fill="none" stroke="#00ff00" d="M775.3099,-121.3134C771.01,-118.0388 766.485,-114.869 762,-112.175 727.1066,-91.2154 684.5821,-73.5284 652.0009,-61.4383"/>
<polygon fill="#00ff00" stroke="#00ff00" points="642.2282,-57.8684 653.1652,-57.0728 646.9247,-59.584 651.6211,-61.2996 651.6211,-61.2996 651.6211,-61.2996 646.9247,-59.584 650.0771,-65.5264 642.2282,-57.8684 642.2282,-57.8684"/>
<text text-anchor="middle" x="761.2825" y="-93.175" font-family="arial" font-size="10.00" fill="#00ff00">Message</text>
</g>
<!-- queueStore -->
<g id="node2" class="node">
<title>queueStore</title>
<path fill="none" stroke="#000000" d="M752.9666,-163.2985C752.9666,-165.6863 736.3976,-167.6259 716,-167.6259 695.6024,-167.6259 679.0334,-165.6863 679.0334,-163.2985 679.0334,-163.2985 679.0334,-124.3515 679.0334,-124.3515 679.0334,-121.9636 695.6024,-120.024 716,-120.024 736.3976,-120.024 752.9666,-121.9636 752.9666,-124.3515 752.9666,-124.3515 752.9666,-163.2985 752.9666,-163.2985"/>
<path fill="none" stroke="#000000" d="M752.9666,-163.2985C752.9666,-160.9107 736.3976,-158.971 716,-158.971 695.6024,-158.971 679.0334,-160.9107 679.0334,-163.2985"/>
<text text-anchor="middle" x="716" y="-147.125" font-family="arial" font-size="11.00" fill="#000000">SMP queue</text>
<text text-anchor="middle" x="716" y="-133.925" font-family="arial" font-size="11.00" fill="#000000">records</text>
</g>
<!-- main -->
<g id="node3" class="node">
<title>main</title>
<polygon fill="none" stroke="#ffa500" points="93.542,-600.5124 74.271,-628.6876 35.729,-628.6876 16.458,-600.5124 35.729,-572.3371 74.271,-572.3371 93.542,-600.5124"/>
<text text-anchor="middle" x="55" y="-603.8124" font-family="arial" font-size="11.00" fill="#000000">main</text>
<text text-anchor="middle" x="55" y="-590.6124" font-family="arial" font-size="11.00" fill="#000000">thread</text>
</g>
<!-- runClient -->
<g id="node5" class="node">
<title>runClient</title>
<polygon fill="none" stroke="#ffa500" points="297.9814,-400.5374 273.4907,-428.7127 224.5093,-428.7127 200.0186,-400.5374 224.5093,-372.3622 273.4907,-372.3622 297.9814,-400.5374"/>
<text text-anchor="middle" x="249" y="-403.8374" font-family="arial" font-size="11.00" fill="#000000">runClient</text>
<text text-anchor="middle" x="249" y="-390.6374" font-family="arial" font-size="11.00" fill="#000000">thread</text>
</g>
<!-- main&#45;&gt;runClient -->
<g id="edge1" class="edge">
<title>main&#45;&gt;runClient</title>
<path fill="none" stroke="#ffa500" stroke-dasharray="5,2" d="M82.4126,-583.8786C98.0501,-573.7433 117.5501,-559.9715 133,-545.4249 168.3831,-512.1106 202.3371,-467.7613 224.2648,-436.9423"/>
<polygon fill="#ffa500" stroke="#ffa500" points="230.1218,-428.629 228.041,-439.3957 227.2421,-432.7164 224.3623,-436.8039 224.3623,-436.8039 224.3623,-436.8039 227.2421,-432.7164 220.6836,-434.2121 230.1218,-428.629 230.1218,-428.629"/>
<text text-anchor="middle" x="203.7235" y="-491.6249" font-family="arial" font-size="10.00" fill="#ffa500">race</text>
</g>
<!-- server -->
<g id="node6" class="node">
<title>server</title>
<polygon fill="none" stroke="#ffa500" points="181.542,-400.5374 162.271,-428.7127 123.729,-428.7127 104.458,-400.5374 123.729,-372.3622 162.271,-372.3622 181.542,-400.5374"/>
<text text-anchor="middle" x="143" y="-403.8374" font-family="arial" font-size="11.00" fill="#000000">server</text>
<text text-anchor="middle" x="143" y="-390.6374" font-family="arial" font-size="11.00" fill="#000000">thread</text>
</g>
<!-- main&#45;&gt;server -->
<g id="edge2" class="edge">
<title>main&#45;&gt;server</title>
<path fill="none" stroke="#ffa500" stroke-dasharray="5,2" d="M67.4216,-572.285C82.8393,-537.2492 109.19,-477.3686 126.3476,-438.379"/>
<polygon fill="#ffa500" stroke="#ffa500" points="130.4994,-428.9443 130.5904,-439.9098 128.4855,-433.5208 126.4715,-438.0972 126.4715,-438.0972 126.4715,-438.0972 128.4855,-433.5208 122.3527,-436.2847 130.4994,-428.9443 130.4994,-428.9443"/>
<text text-anchor="middle" x="119.7235" y="-491.6249" font-family="arial" font-size="10.00" fill="#ffa500">race</text>
</g>
<!-- ss -->
<g id="node4" class="node">
<title>ss</title>
<polygon fill="none" stroke="#0000ff" points="218.4518,-618.5124 111.5482,-618.5124 111.5482,-582.5124 218.4518,-582.5124 218.4518,-618.5124"/>
<text text-anchor="middle" x="165" y="-597.2124" font-family="arial" font-size="11.00" fill="#000000">server TCP socket</text>
</g>
<!-- ss&#45;&gt;runClient -->
<g id="edge3" class="edge">
<title>ss&#45;&gt;runClient</title>
<path fill="none" stroke="#0000ff" d="M190.6135,-582.3228C202.2946,-572.6811 215.2249,-559.8501 223,-545.4249 240.8424,-512.3216 246.7634,-469.4031 248.5729,-438.8905"/>
<polygon fill="#0000ff" stroke="#0000ff" points="249.0542,-428.6517 253.0796,-438.852 248.8194,-433.6461 248.5845,-438.6406 248.5845,-438.6406 248.5845,-438.6406 248.8194,-433.6461 244.0895,-438.4293 249.0542,-428.6517 249.0542,-428.6517"/>
</g>
<!-- cs -->
<g id="node8" class="node">
<title>cs</title>
<polygon fill="none" stroke="#0000ff" points="552.8099,-512.6249 395.1901,-512.6249 395.1901,-476.6249 552.8099,-476.6249 552.8099,-512.6249"/>
<text text-anchor="middle" x="474" y="-491.3249" font-family="arial" font-size="11.00" fill="#000000">client connection TCP socket</text>
</g>
<!-- runClient&#45;&gt;cs -->
<g id="edge5" class="edge">
<title>runClient&#45;&gt;cs</title>
<path fill="none" stroke="#0000ff" stroke-dasharray="5,2" d="M272.5995,-428.901C283.0081,-439.7165 296.0409,-451.2129 309.985,-458.6249 332.9376,-470.8253 359.7984,-478.8942 385.0605,-484.2295"/>
<polygon fill="#0000ff" stroke="#0000ff" points="395.0232,-486.2129 384.337,-488.6737 390.1195,-485.2366 385.2157,-484.2603 385.2157,-484.2603 385.2157,-484.2603 390.1195,-485.2366 386.0944,-479.8469 395.0232,-486.2129 395.0232,-486.2129"/>
<text text-anchor="middle" x="326.5075" y="-449.6249" font-family="arial" font-size="10.00" fill="#0000ff">connect</text>
</g>
<!-- receive -->
<g id="node9" class="node">
<title>receive</title>
<polygon fill="none" stroke="#ffa500" points="422.0134,-290.3624 401.0067,-318.5377 358.9933,-318.5377 337.9866,-290.3624 358.9933,-262.1872 401.0067,-262.1872 422.0134,-290.3624"/>
<text text-anchor="middle" x="380" y="-293.6624" font-family="arial" font-size="11.00" fill="#000000">receive</text>
<text text-anchor="middle" x="380" y="-280.4624" font-family="arial" font-size="11.00" fill="#000000">thread</text>
</g>
<!-- runClient&#45;&gt;receive -->
<g id="edge6" class="edge">
<title>runClient&#45;&gt;receive</title>
<path fill="none" stroke="#ffa500" stroke-dasharray="5,2" d="M242.4559,-372.1449C241.3023,-360.1277 242.3132,-346.5931 249.553,-336.4499 259.7013,-322.232 300.429,-309.2925 333.711,-300.7686"/>
<polygon fill="#ffa500" stroke="#ffa500" points="343.7588,-298.2684 335.1413,-305.05 338.9067,-299.4758 334.0547,-300.6831 334.0547,-300.6831 334.0547,-300.6831 338.9067,-299.4758 332.9681,-296.3163 343.7588,-298.2684 343.7588,-298.2684"/>
<text text-anchor="middle" x="258.7235" y="-339.4499" font-family="arial" font-size="10.00" fill="#ffa500">race</text>
</g>
<!-- client -->
<g id="node10" class="node">
<title>client</title>
<polygon fill="none" stroke="#ffa500" points="612.542,-290.3624 593.271,-318.5377 554.729,-318.5377 535.458,-290.3624 554.729,-262.1872 593.271,-262.1872 612.542,-290.3624"/>
<text text-anchor="middle" x="574" y="-293.6624" font-family="arial" font-size="11.00" fill="#000000">client</text>
<text text-anchor="middle" x="574" y="-280.4624" font-family="arial" font-size="11.00" fill="#000000">thread</text>
</g>
<!-- runClient&#45;&gt;client -->
<g id="edge7" class="edge">
<title>runClient&#45;&gt;client</title>
<path fill="none" stroke="#ffa500" stroke-dasharray="5,2" d="M277.06,-376.3985C296.597,-360.1633 321.2685,-340.9357 333.553,-336.4499 414.4215,-306.9205 446.2239,-356.2313 527,-326.4499 532.1882,-324.5371 537.2696,-321.8366 542.0605,-318.7636"/>
<polygon fill="#ffa500" stroke="#ffa500" points="550.3305,-312.9195 544.7608,-322.3656 546.2472,-315.8051 542.1638,-318.6906 542.1638,-318.6906 542.1638,-318.6906 546.2472,-315.8051 539.5668,-315.0156 550.3305,-312.9195 550.3305,-312.9195"/>
<text text-anchor="middle" x="342.7235" y="-339.4499" font-family="arial" font-size="10.00" fill="#ffa500">race</text>
</g>
<!-- send -->
<g id="node11" class="node">
<title>send</title>
<polygon fill="none" stroke="#ffa500" points="517.542,-290.3624 498.271,-318.5377 459.729,-318.5377 440.458,-290.3624 459.729,-262.1872 498.271,-262.1872 517.542,-290.3624"/>
<text text-anchor="middle" x="479" y="-293.6624" font-family="arial" font-size="11.00" fill="#000000">send</text>
<text text-anchor="middle" x="479" y="-280.4624" font-family="arial" font-size="11.00" fill="#000000">thread</text>
</g>
<!-- runClient&#45;&gt;send -->
<g id="edge8" class="edge">
<title>runClient&#45;&gt;send</title>
<path fill="none" stroke="#ffa500" stroke-dasharray="5,2" d="M260.5478,-372.3313C267.4946,-359.1125 277.7098,-344.526 291.553,-336.4499 318.388,-320.7945 401.9033,-337.3399 431,-326.4499 436.5046,-324.3897 441.9126,-321.4869 446.9995,-318.2123"/>
<polygon fill="#ffa500" stroke="#ffa500" points="455.2178,-312.4339 449.6258,-321.8669 451.1276,-315.3098 447.0375,-318.1857 447.0375,-318.1857 447.0375,-318.1857 451.1276,-315.3098 444.4492,-314.5046 455.2178,-312.4339 455.2178,-312.4339"/>
<text text-anchor="middle" x="300.7235" y="-339.4499" font-family="arial" font-size="10.00" fill="#ffa500">race</text>
</g>
<!-- inq -->
<g id="node12" class="node">
<title>inq</title>
<polygon fill="none" stroke="#000000" points="375.5723,-56.0875 330.4277,-56.0875 330.4277,-32.0875 375.5723,-32.0875 375.5723,-26.0875 393.5723,-44.0875 375.5723,-62.0875 375.5723,-56.0875"/>
<text text-anchor="middle" x="362" y="-47.3875" font-family="arial" font-size="11.00" fill="#000000">receive</text>
<text text-anchor="middle" x="362" y="-34.1875" font-family="arial" font-size="11.00" fill="#000000">TBQueue</text>
</g>
<!-- server&#45;&gt;inq -->
<g id="edge9" class="edge">
<title>server&#45;&gt;inq</title>
<path fill="none" stroke="#000000" d="M160.3676,-372.2694C201.6259,-305.1164 305.0826,-136.7276 345.4771,-70.9806"/>
<polygon fill="#000000" stroke="#000000" points="350.7453,-62.4059 349.3446,-73.2819 348.1279,-66.6661 345.5104,-70.9262 345.5104,-70.9262 345.5104,-70.9262 348.1279,-66.6661 341.6763,-68.5705 350.7453,-62.4059 350.7453,-62.4059"/>
<text text-anchor="middle" x="270.5535" y="-223.275" font-family="arial" font-size="10.00" fill="#000000">END</text>
</g>
<!-- subscribedQ -->
<g id="node7" class="node">
<title>subscribedQ</title>
<polygon fill="none" stroke="#000000" points="305.6775,-612.5124 254.3225,-612.5124 254.3225,-618.5124 236.3225,-600.5124 254.3225,-582.5124 254.3225,-588.5124 305.6775,-588.5124 305.6775,-612.5124"/>
<text text-anchor="middle" x="271" y="-603.8124" font-family="arial" font-size="11.00" fill="#000000">subscribed</text>
<text text-anchor="middle" x="271" y="-590.6124" font-family="arial" font-size="11.00" fill="#000000">TBQueue</text>
</g>
<!-- subscribedQ&#45;&gt;server -->
<g id="edge4" class="edge">
<title>subscribedQ&#45;&gt;server</title>
<path fill="none" stroke="#000000" d="M266.5979,-582.0921C259.9663,-556.7857 245.7163,-510.5487 223,-476.6249 210.3763,-457.773 192.3233,-440.0878 176.5604,-426.5205"/>
<polygon fill="#000000" stroke="#000000" points="168.5671,-419.8168 179.1209,-422.7948 172.3982,-423.0298 176.2292,-426.2428 176.2292,-426.2428 176.2292,-426.2428 172.3982,-423.0298 173.3375,-429.6907 168.5671,-419.8168 168.5671,-419.8168"/>
</g>
<!-- cs&#45;&gt;receive -->
<g id="edge10" class="edge">
<title>cs&#45;&gt;receive</title>
<path fill="none" stroke="#0000ff" d="M465.6581,-476.498C450.379,-443.2963 417.6111,-372.0915 397.3247,-328.0092"/>
<polygon fill="#0000ff" stroke="#0000ff" points="393.0726,-318.7693 401.3411,-325.9723 395.1629,-323.3114 397.2532,-327.8535 397.2532,-327.8535 397.2532,-327.8535 395.1629,-323.3114 393.1653,-329.7348 393.0726,-318.7693 393.0726,-318.7693"/>
</g>
<!-- receive&#45;&gt;inq -->
<g id="edge11" class="edge">
<title>receive&#45;&gt;inq</title>
<path fill="none" stroke="#0000ff" d="M377.9352,-262.1116C374.4741,-214.757 367.5409,-119.8979 364.0738,-72.4613"/>
<polygon fill="#0000ff" stroke="#0000ff" points="363.33,-62.2842 368.547,-71.9295 363.6945,-67.2709 364.059,-72.2576 364.059,-72.2576 364.059,-72.2576 363.6945,-67.2709 359.571,-72.5856 363.33,-62.2842 363.33,-62.2842"/>
</g>
<!-- client&#45;&gt;msgQueues -->
<g id="edge17" class="edge">
<title>client&#45;&gt;msgQueues</title>
<path fill="none" stroke="#00ff00" d="M615.7447,-277.4263C656.3868,-263.3055 718.2574,-237.6106 762,-200.275 769.9504,-193.4891 777.0327,-184.7227 782.9078,-176.1309"/>
<polygon fill="#00ff00" stroke="#00ff00" points="606.1466,-280.6832 614.1703,-273.2085 610.8815,-279.0765 615.6163,-277.4699 615.6163,-277.4699 615.6163,-277.4699 610.8815,-279.0765 617.0623,-281.7312 606.1466,-280.6832 606.1466,-280.6832"/>
<polygon fill="#00ff00" stroke="#00ff00" points="788.3681,-167.6771 786.7225,-178.5188 785.6553,-171.8771 782.9424,-176.0772 782.9424,-176.0772 782.9424,-176.0772 785.6553,-171.8771 779.1624,-173.6357 788.3681,-167.6771 788.3681,-167.6771"/>
<text text-anchor="middle" x="763.2775" y="-235.275" font-family="arial" font-size="10.00" fill="#000000">SEND,</text>
<text text-anchor="middle" x="763.2775" y="-223.275" font-family="arial" font-size="10.00" fill="#000000">SUB,</text>
<text text-anchor="middle" x="763.2775" y="-211.275" font-family="arial" font-size="10.00" fill="#000000">ACK</text>
</g>
<!-- client&#45;&gt;queueStore -->
<g id="edge15" class="edge">
<title>client&#45;&gt;queueStore</title>
<path fill="none" stroke="#000000" d="M609.8863,-268.3247C620.1282,-261.2548 630.9539,-252.9826 640,-244.275 661.2301,-223.8393 681.1673,-197.0861 695.2906,-176.3279"/>
<polygon fill="#000000" stroke="#000000" points="601.4722,-273.9638 607.2738,-264.6583 605.6257,-271.1801 609.7791,-268.3964 609.7791,-268.3964 609.7791,-268.3964 605.6257,-271.1801 612.2844,-272.1346 601.4722,-273.9638 601.4722,-273.9638"/>
<polygon fill="#000000" stroke="#000000" points="700.9743,-167.8345 699.1526,-178.648 698.1935,-171.9899 695.4127,-176.1453 695.4127,-176.1453 695.4127,-176.1453 698.1935,-171.9899 691.6729,-173.6426 700.9743,-167.8345 700.9743,-167.8345"/>
</g>
<!-- client&#45;&gt;subscribedQ -->
<g id="edge21" class="edge">
<title>client&#45;&gt;subscribedQ</title>
<path fill="none" stroke="#000000" d="M548.6699,-309.8896C541.6619,-315.2708 534.0458,-321.0992 527,-326.4499 456.1732,-380.2374 427.111,-381.0795 367,-446.6249 330.9316,-485.954 300.5471,-540.6501 284.1272,-573.1325"/>
<polygon fill="#000000" stroke="#000000" points="279.4889,-582.4585 279.9129,-571.5008 281.7155,-577.9816 283.9421,-573.5048 283.9421,-573.5048 283.9421,-573.5048 281.7155,-577.9816 287.9713,-575.5087 279.4889,-582.4585 279.4889,-582.4585"/>
<text text-anchor="middle" x="391.7185" y="-449.6249" font-family="arial" font-size="10.00" fill="#000000">(rId, Client)</text>
</g>
<!-- client&#45;&gt;subscriber -->
<g id="edge16" class="edge">
<title>client&#45;&gt;subscriber</title>
<path fill="none" stroke="#ffa500" stroke-dasharray="5,2" d="M577.0972,-262.1116C581.9467,-217.8786 591.3402,-132.1977 596.796,-82.4333"/>
<polygon fill="#ffa500" stroke="#ffa500" points="597.8974,-72.3872 601.2807,-82.818 597.3525,-77.3574 596.8075,-82.3276 596.8075,-82.3276 596.8075,-82.3276 597.3525,-77.3574 592.3343,-81.8371 597.8974,-72.3872 597.8974,-72.3872"/>
<text text-anchor="middle" x="619.1205" y="-146.825" font-family="arial" font-size="10.00" fill="#000000">1 fork per</text>
<text text-anchor="middle" x="619.1205" y="-134.825" font-family="arial" font-size="10.00" fill="#000000">SMP queue</text>
</g>
<!-- outq -->
<g id="node14" class="node">
<title>outq</title>
<polygon fill="none" stroke="#000000" points="474.5723,-56.0875 429.4277,-56.0875 429.4277,-62.0875 411.4277,-44.0875 429.4277,-26.0875 429.4277,-32.0875 474.5723,-32.0875 474.5723,-56.0875"/>
<text text-anchor="middle" x="443" y="-47.3875" font-family="arial" font-size="11.00" fill="#000000">send</text>
<text text-anchor="middle" x="443" y="-34.1875" font-family="arial" font-size="11.00" fill="#000000">TBQueue</text>
</g>
<!-- client&#45;&gt;outq -->
<g id="edge18" class="edge">
<title>client&#45;&gt;outq</title>
<path fill="none" stroke="#0000ff" d="M569.0363,-262.148C560.7382,-221.1225 540.9769,-144.0325 502,-90.175 496.236,-82.2103 488.7272,-74.9287 481.0081,-68.6219"/>
<polygon fill="#0000ff" stroke="#0000ff" points="472.622,-62.1747 483.2926,-64.7021 476.5859,-65.2222 480.5499,-68.2697 480.5499,-68.2697 480.5499,-68.2697 476.5859,-65.2222 477.8071,-71.8372 472.622,-62.1747 472.622,-62.1747"/>
</g>
<!-- send&#45;&gt;cs -->
<g id="edge20" class="edge">
<title>send&#45;&gt;cs</title>
<path fill="none" stroke="#0000ff" d="M478.3046,-318.7693C477.35,-357.7707 475.6436,-427.4808 474.6911,-466.3911"/>
<polygon fill="#0000ff" stroke="#0000ff" points="474.4437,-476.498 470.1899,-466.3908 474.5661,-471.4995 474.6885,-466.501 474.6885,-466.501 474.6885,-466.501 474.5661,-471.4995 479.1872,-466.6111 474.4437,-476.498 474.4437,-476.498"/>
</g>
<!-- inq&#45;&gt;client -->
<g id="edge12" class="edge">
<title>inq&#45;&gt;client</title>
<path fill="none" stroke="#0000ff" d="M377.6198,-62.2326C413.1933,-103.5575 500.6514,-205.1552 545.8859,-257.703"/>
<polygon fill="#0000ff" stroke="#0000ff" points="552.5056,-265.393 542.5712,-260.75 549.2436,-261.6036 545.9816,-257.8142 545.9816,-257.8142 545.9816,-257.8142 549.2436,-261.6036 549.3921,-254.8784 552.5056,-265.393 552.5056,-265.393"/>
</g>
<!-- subscriber&#45;&gt;outq -->
<g id="edge14" class="edge">
<title>subscriber&#45;&gt;outq</title>
<path fill="none" stroke="#0000ff" d="M546.4214,-44.0875C526.1863,-44.0875 503.5804,-44.0875 484.6766,-44.0875"/>
<polygon fill="#0000ff" stroke="#0000ff" points="474.5939,-44.0875 484.594,-39.5876 479.5939,-44.0875 484.5939,-44.0876 484.5939,-44.0876 484.5939,-44.0876 479.5939,-44.0875 484.5939,-48.5876 474.5939,-44.0875 474.5939,-44.0875"/>
<text text-anchor="middle" x="510.528" y="-50.0875" font-family="arial" font-size="10.00" fill="#0000ff">MSG</text>
</g>
<!-- outq&#45;&gt;send -->
<g id="edge19" class="edge">
<title>outq&#45;&gt;send</title>
<path fill="none" stroke="#0000ff" d="M445.66,-62.2842C451.4907,-102.1724 465.5041,-198.0375 473.381,-251.9232"/>
<polygon fill="#0000ff" stroke="#0000ff" points="474.8703,-262.1116 468.9712,-252.8677 474.1471,-257.1642 473.4239,-252.2167 473.4239,-252.2167 473.4239,-252.2167 474.1471,-257.1642 477.8765,-251.5658 474.8703,-262.1116 474.8703,-262.1116"/>
</g>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 21 KiB

+240
View File
@@ -0,0 +1,240 @@
# Subscription changes processed in the client thread
## Decisions
- Each subscription change is processed by `processClientSub` under `uninterruptibleMask_`, in the client thread of the connection.
- `serverThread` and `subQ` are removed.
- `processClientSub` takes the place of `serverThread` in `smpServer`; its branch for a disconnected client is removed.
- SUB, from clients with and without a service certificate: the subscriber is registered in `prepareBatchSubs`, after the queue associations and before `tryPeekMsgs`, except for a service client when the queue is associated with its service and the connection has a service count, as on master.
- Duplicate SUB: `delivered` of the subscription is reset in `prepareBatchSubs`, before `tryPeekMsgs`.
- SUB of a blocked queue, or of a queue with a failed association: the subscriber is registered only by a later successful SUB.
- For a new subscription, the first message is marked as delivered once, and it is sent only where it is marked, in one of these:
- the MSG pushed after SEND
- the SUB response
- the MSG sent by `deliverServiceMessages` after SUBS
- After a duplicate SUB, a message pushed before `delivered` is reset is marked and sent again, as on master.
- `qSub` is counted for a new subscription when a message is returned by `tryPeekMsgs`, with these exceptions:
- two SUBs of one queue in one block
- an `updateQueueTime` error after the registration
- The service counts of a previous service subscriber are decreased for one queue only while it is registered for the service.
- The subscription changes of each of these commands are processed during that command, in the client thread:
- SUBS
- NSUB
- NSUBS
- NEW with subscription
- NKEY
- NDEL
- DEL
- The SMP server design diagram and its README section are removed.
## Composition
Processing of one subscription change. Green: added by this pull request. Red: removed by it.
```mermaid
flowchart LR
C[Client thread of a connection]
Q[subQ]
T[serverThread]
X[Server clients]
P[processClientSub<br>under uninterruptibleMask_]
U[updateSubscribers<br>one STM transaction]
V[endPreviousSubscriptions<br>later STM transactions]
R[queueSubscribers<br>serviceSubscribers<br>totalServiceSubs<br>subClients<br>maps and counts of clients]
E[pendingEvents]
S[sendPendingEvtsThread]
C -- "removed - change written" --> Q
Q -- "removed - change read" --> T
T -- "removed - client read by getServerClient" --> X
T -- "removed - run" --> U
T -- "removed - run" --> V
C -- "added - processClientSub called" --> P
P -- "added - run" --> U
P -- "added - run" --> V
U -- "client registered<br>client added to subClients<br>totalServiceSubs increased<br>entries of the previous subscribers removed or replaced<br>count of the previous service connection swapped to 0<br>client removed from subClients after a deletion without other subscriptions" --> R
V -- "previous subscriptions removed from client maps<br>previous client removed from subClients without other subscriptions<br>counts decreased<br>delivery threads stopped" --> R
V -- "END<br>ENDS<br>DELD" --> E
E -- "events read" --> S
classDef removed fill:#f4cccc,stroke:#cc0000
classDef added fill:#d9ead3,stroke:#008800
class Q,T removed
class P added
linkStyle 0,1,2,3,4 stroke:#cc0000
linkStyle 5,6,7 stroke:#008800
```
## SUB and SEND
SUB of an empty queue and a SEND to it right after `tryPeekMsgs`. Green: added by this pull request. Red: removed by it.
```mermaid
sequenceDiagram
participant R as Recipient agent
participant C as Client thread of R
participant Q as Message store
participant T as serverThread, removed
participant S as Client thread of sender
R->>C: SUB q, signature verified in receive
rect rgba(0, 160, 0, 0.18)
C->>C: added - prepareBatchSubs, subscribeQueue: Sub of q inserted
C->>C: added - updateSubscribers under uninterruptibleMask_: R registered in queueSubscribers
C->>C: added - endPreviousSubscriptions: END queued for the previous subscriber
end
C->>Q: prepareBatchSubs: tryPeekMsgs q
Q-->>C: empty queue
S->>Q: SEND m: writeMsg, the queue was empty
rect rgba(220, 0, 0, 0.18)
S->>S: removed - tryDeliverMessage: R not found, m not pushed to R
end
rect rgba(0, 160, 0, 0.18)
S-->>R: added - tryDeliverMessage: R found, MSG m pushed and marked as delivered
end
rect rgba(220, 0, 0, 0.18)
C->>C: removed - subscribeQueueAndDeliver: Sub of q inserted
C->>T: removed - CSClient q written to subQ
T->>T: removed - updateSubscribers: R registered in queueSubscribers
T->>T: removed - endPreviousSubscriptions: END queued for the previous subscriber
end
C-->>R: SOK
rect rgba(220, 0, 0, 0.18)
Note over R,Q: removed - m returned to R only in the response to the next SUB
end
rect rgba(0, 160, 0, 0.18)
Note over R,Q: added - when m is written after the registration and before tryPeekMsgs, m is returned by tryPeekMsgs, marked as delivered once, after SEND or for the SUB response, and sent only where it is marked
end
```
## Duplicate SUB
SUB of a queue that is subscribed in this connection, with message m delivered and not acknowledged. Green: added by this pull request. Red: removed by it.
```mermaid
sequenceDiagram
participant R as Recipient agent
participant C as Client thread of R
participant Q as Message store
R->>C: SUB q
rect rgba(0, 160, 0, 0.18)
C->>C: added - prepareBatchSubs, subscribeQueue: delivered of q reset
end
C->>Q: prepareBatchSubs: tryPeekMsgs q
Q-->>C: m
rect rgba(220, 0, 0, 0.18)
C->>C: removed - subscribeQueueAndDeliver: delivered of q reset
C->>C: removed - deliver: m marked by setDelivered and included in the SUB response
end
rect rgba(0, 160, 0, 0.18)
C->>C: added - deliver: m included in the SUB response only when it is marked by trySetDelivered
end
C-->>R: SOK, MSG m
```
## Duplicate SUB and SEND
Duplicate SUB of an empty queue, with nothing marked as delivered, and a SEND of m to it before `tryPeekMsgs`. Green: added by this pull request. Red: removed by it.
```mermaid
sequenceDiagram
participant R as Recipient agent
participant C as Client thread of R
participant Q as Message store
participant S as Client thread of sender
R->>C: SUB q
rect rgba(0, 160, 0, 0.18)
C->>C: added - prepareBatchSubs, subscribeQueue: delivered of q reset
end
S->>Q: SEND m: writeMsg, the queue was empty
S-->>R: tryDeliverMessage: MSG m pushed and marked as delivered
C->>Q: prepareBatchSubs: tryPeekMsgs q
Q-->>C: m
rect rgba(220, 0, 0, 0.18)
C->>C: removed - subscribeQueueAndDeliver: delivered of q reset
C-->>R: removed - SOK, MSG m again
end
rect rgba(0, 160, 0, 0.18)
C-->>R: added - SOK only, as m is marked already
end
```
## SUBS
SUBS of a service connection, and a SEND of m to an empty queue q of the service while its messages are read. Green: added by this pull request. Red: removed by it.
```mermaid
sequenceDiagram
participant C as Client thread of the service connection
participant T as serverThread, removed
participant F as deliverServiceMessages
participant S as Client thread of sender
rect rgba(220, 0, 0, 0.18)
C->>T: removed - CSService written to subQ, service registered by serverThread
end
rect rgba(0, 160, 0, 0.18)
C->>C: added - processClientSub: service registered
end
C->>F: deliverServiceMessages forked
S->>S: SEND m: writeMsg, the queue was empty
F->>F: foldRcvServiceMessages: m of q read, Sub of q inserted
S-->>C: tryDeliverMessage: Sub of q found, MSG m pushed and marked as delivered
rect rgba(220, 0, 0, 0.18)
F-->>C: removed - m marked by setDelivered, MSG m sent again
end
rect rgba(0, 160, 0, 0.18)
F->>F: added - trySetDelivered: m marked already, MSG omitted, counted in rcvServicesSubDuplicate
end
```
## Service counts
A SUB by which the association of queue q with service S is removed, and SUBS from a new connection N of S. O is the previous connection of S. Green: added by this pull request. Red: removed by it.
```mermaid
sequenceDiagram
participant D as Client thread of the SUB
participant N as Client thread of N
participant G as Registries
D->>G: updateSubscribers: O recorded for CSAEndServiceSub S q
N->>G: updateSubscribers: N registered for S, the count of O swapped to 0
N->>G: endPreviousSubscriptions: the swapped count subtracted from totalServiceSubs
rect rgba(220, 0, 0, 0.18)
D->>G: removed - endServiceQueueSub: the count of O and totalServiceSubs decreased for q again
end
rect rgba(0, 160, 0, 0.18)
D->>G: added - endServiceQueueSub: O not registered for S, counts unchanged
end
```
## Mechanics
- `Server.hs`:
- `processClientSub`: `serverThread` in its place, for one change of the client of the calling thread, under `uninterruptibleMask_`.
- `CSAEndServiceSub`: the service ID added.
- `endServiceQueueSub`: the counts decreased only while the previous client is registered for the service.
- `smpServer`: the two `serverThread` threads removed.
- `runClient`: `processSubs`, the pair of `processClientSub` for message and notification subscribers, passed through `runClientTransport` to `client`.
- `client`: `processSub` and `processNtfSub` from the pair.
- `prepareBatchSubs`: one `runExceptT` block:
- queue associations
- `subscribeQueue` for each SUB of a queue that is not blocked, when its association is unchanged or changed successfully
- `tryPeekMsgs`
- the `subscribeQueue` results returned as the fourth map of the batch result
- `subscribeQueue`: the first part of the former `subscribeQueueAndDeliver`.
- `deliverSubMsg`: the second part of the former `subscribeQueueAndDeliver`; the `subscribeQueue` result passed to `deliver`.
- `sharedSubscribeQueue`: moved from `processCommand` to `client`; the service ID of the client and the queue ID added as parameters; `processSub` or `processNtfSub` passed in place of the server subscribers.
- `processSub` or `processNtfSub` called with the client and the subscription change where the change was written to `subQ`:
- `sharedSubscribeService`
- `subscribeNewQueue`
- `addQueueNotifier_`
- `deleteQueueNotifier_`
- `delQueueAndMsgs`
- `deliver`: `qSub` counted before `trySetDelivered`; the first message included only when it is marked by `trySetDelivered`.
- `deliverServiceMessages`: MSG sent only when the message is marked by `trySetDelivered`.
- `Env/STM.hs`: `subQ` removed from `ServerSubscribers`.
- `README.md`: the section "SMP server design" removed.
- `design/server.gv`, `design/server.svg`: removed.
- `tests/ServerTests.hs`:
- `should deliver message sent in the same block as SUB`
- `should deliver message sent in the same block as SUB to the new connection`
- `should deliver message once for two SUBs in the same block`
- `should deliver message sent in the same block as service SUB`
- `should deliver message after SUB while delivery waits for full send queue`: the message recorded before or after SOK.
+168 -167
View File
@@ -61,7 +61,7 @@ import qualified Data.ByteString.Char8 as B
import qualified Data.ByteString.Lazy.Char8 as LB
import Data.Constraint (Dict (..))
import Data.Dynamic (toDyn)
import Data.Either (fromRight, partitionEithers)
import Data.Either (fromRight, isRight, partitionEithers)
import Data.Foldable (foldrM)
import Data.Functor (($>), (<&>))
import Data.IORef
@@ -163,10 +163,10 @@ runSMPServerBlocking started cfg attachHTTP_ = newEnv cfg >>= runReaderT (smpSer
type M s a = ReaderT (Env s) IO a
type AttachHTTP = Socket -> TLS.Context -> IO ()
-- actions used in serverThread to reduce STM transaction scope
-- actions used in processClientSub to reduce STM transaction scope
data ClientSubAction
= CSAEndSub QueueId -- end single direct queue subscription
| CSAEndServiceSub QueueId -- end service subscription to one queue
| CSAEndServiceSub ServiceId QueueId -- end service subscription to one queue
| CSADecreaseSubs (Int64, IdsHash) -- reduce service subscriptions when cancelling. Fixed number is used to correctly handle race conditions when service resubscribes
type PrevClientSub s = (Client s, ClientSubAction, (EntityId, BrokerMsg))
@@ -185,9 +185,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
stopServer s
liftIO $ exitSuccess
raceAny_
( serverThread "server subscribers" s subscribers subscriptions serviceSubsCount (Just cancelSub)
: serverThread "server ntfSubscribers" s ntfSubscribers ntfSubscriptions ntfServiceSubsCount Nothing
: deliverNtfsThread s
( deliverNtfsThread s
: sendPendingEvtsThread s
: receiveFromProxyAgent pa
: expireNtfsThread cfg
@@ -248,70 +246,49 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
asks (smpAgent . proxyAgent) >>= liftIO . closeSMPClientAgent
asks namesEnv >>= liftIO . mapM_ closeNamesEnv
serverThread ::
forall sub. String ->
processClientSub ::
forall sub.
Server s ->
(Server s -> ServerSubscribers s) ->
(Client s -> TMap QueueId sub) ->
(Client s -> TVar (Int64, IdsHash)) ->
Maybe (sub -> IO ()) ->
M s ()
serverThread label srv srvSubscribers clientSubs clientServiceSubs unsub_ = do
labelMyThread label
liftIO . forever $ do
-- Reading clients outside of `updateSubscribers` transaction to avoid transaction re-evaluation on each new connected client.
-- In case client disconnects during the transaction (its `connected` property is read),
-- the transaction will still be re-evaluated, and the client won't be stored as subscribed.
sub@(_, clntId) <- atomically $ readTQueue subQ
c_ <- getServerClient clntId srv
atomically (updateSubscribers c_ sub)
Client s -> ClientSub -> IO ()
processClientSub srv srvSubscribers clientSubs clientServiceSubs unsub_ clnt sub =
uninterruptibleMask_ $
atomically (updateSubscribers clnt sub)
>>= endPreviousSubscriptions
where
ServerSubscribers {subQ, queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents} = srvSubscribers srv
updateSubscribers :: Maybe (Client s) -> (ClientSub, ClientId) -> STM [PrevClientSub s]
updateSubscribers c_ (clntSub, clntId) = case c_ of
Just c@Client {connected} -> ifM (readTVar connected) (updateSubConnected c) updateSubDisconnected
Nothing -> updateSubDisconnected
ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents} = srvSubscribers srv
updateSubscribers :: Client s -> ClientSub -> STM [PrevClientSub s]
updateSubscribers c@Client {clientId = clntId} clntSub = case clntSub of
CSClient qId prevServiceId serviceId_ -> do
modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients
as'' <- if prevServiceId == serviceId_ then pure [] else endServiceSub prevServiceId qId END
case serviceId_ of
Just serviceId -> do
modifyTVar' totalServiceSubs $ addServiceSubs (1, queueIdHash qId) -- server count and IDs hash for all services
as <- endQueueSub qId END
as' <- cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers
pure $ as ++ as' ++ as''
Nothing -> do
as <- prevSub qId END (CSAEndSub qId) =<< upsertSubscribedClient qId c queueSubscribers
pure $ as ++ as''
CSDeleted qId serviceId -> do
removeWhenNoSubs c
as <- endQueueSub qId DELD
as' <- endServiceSub serviceId qId DELD
pure $ as ++ as'
CSService serviceId changedSubs -> do
modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients
modifyTVar' totalServiceSubs $ addServiceSubs changedSubs -- server count and IDs hash for all services
cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers
where
updateSubConnected c = case clntSub of
CSClient qId prevServiceId serviceId_ -> do
modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients
as'' <- if prevServiceId == serviceId_ then pure [] else endServiceSub prevServiceId qId END
case serviceId_ of
Just serviceId -> do
modifyTVar' totalServiceSubs $ addServiceSubs (1, queueIdHash qId) -- server count and IDs hash for all services
as <- endQueueSub qId END
as' <- cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers
pure $ as ++ as' ++ as''
Nothing -> do
as <- prevSub qId END (CSAEndSub qId) =<< upsertSubscribedClient qId c queueSubscribers
pure $ as ++ as''
CSDeleted qId serviceId -> do
removeWhenNoSubs c
as <- endQueueSub qId DELD
as' <- endServiceSub serviceId qId DELD
pure $ as ++ as'
CSService serviceId changedSubs -> do
modifyTVar' subClients $ IS.insert clntId -- add ID to server's subscribed cients
modifyTVar' totalServiceSubs $ addServiceSubs changedSubs -- server count and IDs hash for all services
cancelServiceSubs serviceId =<< upsertSubscribedClient serviceId c serviceSubscribers
updateSubDisconnected = case clntSub of
-- do not insert client if it is already disconnected, but send END/DELD to any other client subscribed to this queue or service
CSClient qId prevServiceId serviceId -> do
as <- endQueueSub qId END
as' <- endServiceSub serviceId qId END
as'' <- if prevServiceId == serviceId then pure [] else endServiceSub prevServiceId qId END
pure $ as ++ as' ++ as''
CSDeleted qId serviceId -> do
as <- endQueueSub qId DELD
as' <- endServiceSub serviceId qId DELD
pure $ as ++ as'
CSService serviceId _ -> cancelServiceSubs serviceId =<< lookupSubscribedClient serviceId serviceSubscribers
endQueueSub :: QueueId -> BrokerMsg -> STM [PrevClientSub s]
endQueueSub qId msg = prevSub qId msg (CSAEndSub qId) =<< lookupDeleteSubscribedClient qId queueSubscribers
endServiceSub :: Maybe ServiceId -> QueueId -> BrokerMsg -> STM [PrevClientSub s]
endServiceSub Nothing _ _ = pure []
endServiceSub (Just serviceId) qId msg = prevSub qId msg (CSAEndServiceSub qId) =<< lookupSubscribedClient serviceId serviceSubscribers
endServiceSub (Just serviceId) qId msg = prevSub qId msg (CSAEndServiceSub serviceId qId) =<< lookupSubscribedClient serviceId serviceSubscribers
prevSub :: QueueId -> BrokerMsg -> ClientSubAction -> Maybe (Client s) -> STM [PrevClientSub s]
prevSub qId msg action =
checkAnotherClient $ \c -> pure [(c, action, (qId, msg))]
@@ -333,7 +310,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
CSAEndSub qId -> atomically (endSub c qId) >>= unsubPrev
-- like endSub, also removes the delivery subscription from the service client's subscriptions map,
-- otherwise the map retains entries for queues unassociated/deleted while the service stays connected.
CSAEndServiceSub qId -> atomically (endServiceQueueSub c qId) >>= unsubPrev
CSAEndServiceSub serviceId qId -> atomically (endServiceQueueSub c serviceId qId) >>= unsubPrev
CSADecreaseSubs changedSubs -> do
atomically $ modifyTVar' totalServiceSubs $ subtractServiceSubs changedSubs
forM_ unsub_ $ \unsub -> atomically (swapTVar (clientSubs c) M.empty) >>= mapM_ unsub
@@ -342,10 +319,11 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
unsubPrev s_ = sequence_ (unsub_ <*> s_)
endSub :: Client s -> QueueId -> STM (Maybe sub)
endSub c qId = TM.lookupDelete qId (clientSubs c) >>= (removeWhenNoSubs c $>)
endServiceQueueSub :: Client s -> QueueId -> STM (Maybe sub)
endServiceQueueSub c qId = do
modifyTVar' (clientServiceSubs c) decrease
modifyTVar' totalServiceSubs decrease
endServiceQueueSub :: Client s -> ServiceId -> QueueId -> STM (Maybe sub)
endServiceQueueSub c serviceId qId = do
whenM (maybe False (sameClientId c) <$> lookupSubscribedClient serviceId serviceSubscribers) $ do
modifyTVar' (clientServiceSubs c) decrease
modifyTVar' totalServiceSubs decrease
endSub c qId
where
decrease = subtractServiceSubs (1, queueIdHash qId)
@@ -733,6 +711,7 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
runClient :: Transport c => X.CertificateChain -> C.APrivateSignKey -> TProxy c 'TServer -> c 'TServer -> M s ()
runClient srvCert srvSignKey tp h = do
ms <- asks msgStore
srv <- asks server
g <- asks random
idSize <- asks $ queueIdBytes . config
kh <- asks serverIdentity
@@ -741,8 +720,10 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
let serverInfo = LB.toStrict . J.encode <$> information
labelMyThread $ "smp handshake for " <> transportName tp
liftIO (timeout smpHandshakeTimeout . runExceptT $ smpServerHandshake srvCert srvSignKey h ks kh smpServerVRange serverInfo $ getClientService ms g idSize) >>= \case
Just (Right th) -> runClientTransport th
Just (Right th) -> runClientTransport (processSubs srv) th
_ -> pure ()
where
processSubs srv = (processClientSub srv subscribers subscriptions serviceSubsCount (Just cancelSub), processClientSub srv ntfSubscribers ntfSubscriptions ntfServiceSubsCount Nothing)
getClientService :: s -> TVar ChaChaDRG -> Int -> SMPServiceRole -> X.CertificateChain -> XV.Fingerprint -> ExceptT TransportError IO ServiceId
getClientService ms g idSize role cert fp = do
@@ -1059,8 +1040,8 @@ smpServer started cfg@ServerConfig {transports, transportConfig = tCfg, startOpt
logError "Unauthorized control port command"
hPutStrLn h "AUTH"
runClientTransport :: forall c s. (Transport c, MsgStoreClass s) => THandleSMP c 'TServer -> M s ()
runClientTransport h@THandle {params = thParams@THandleParams {sessionId}} = do
runClientTransport :: forall c s. (Transport c, MsgStoreClass s) => (Client s -> ClientSub -> IO (), Client s -> ClientSub -> IO ()) -> THandleSMP c 'TServer -> M s ()
runClientTransport processSubs h@THandle {params = thParams@THandleParams {sessionId}} = do
q <- asks $ tbqSize . config
ts <- liftIO getSystemTime
nextClientId <- asks clientSeq
@@ -1076,7 +1057,7 @@ runClientTransport h@THandle {params = thParams@THandleParams {sessionId}} = do
expCfg <- asks $ inactiveClientExpiration . config
th <- newMVar h -- put TH under a fair lock to interleave messages and command responses
labelMyThread . B.unpack $ "client $" <> encode sessionId
raceAny_ $ [liftIO $ send th c, liftIO $ sendMsg th c, client s ms c, receive h ms c] <> disconnectThread_ c s expCfg
raceAny_ $ [liftIO $ send th c, liftIO $ sendMsg th c, client processSubs s ms c, receive h ms c] <> disconnectThread_ c s expCfg
disconnectThread_ :: Client s -> Server s -> Maybe ExpirationConfig -> [M s ()]
disconnectThread_ c s (Just expCfg) = [liftIO $ disconnectTransport h (rcvActiveAt c) (sndActiveAt c) expCfg (noSubscriptions c s)]
disconnectThread_ _ _ _ = []
@@ -1405,11 +1386,12 @@ forkClient Client {endThreads, endThreadSeq} label action = do
action `finally` atomically (modifyTVar' endThreads $ IM.delete tId)
mkWeakThreadId t >>= atomically . modifyTVar' endThreads . IM.insert tId
client :: forall s. MsgStoreClass s => Server s -> s -> Client s -> M s ()
client :: forall s. MsgStoreClass s => (Client s -> ClientSub -> IO (), Client s -> ClientSub -> IO ()) -> Server s -> s -> Client s -> M s ()
client
Server {subscribers, ntfSubscribers}
(processSub, processNtfSub)
Server {subscribers}
ms
clnt@Client {clientId, rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do
clnt@Client {rcvQ, sndQ, msgQ, clientTHParams = thParams'@THandleParams {sessionId}, procThreads} = do
labelMyThread . B.unpack $ "client $" <> encode sessionId <> " commands"
stats <- asks serverStats
authDelay <- asks $ authErrorDelay . config
@@ -1427,25 +1409,100 @@ client
prepareBatchSubs ::
Maybe ServiceId ->
NonEmpty (VerifiedTransmission s) ->
M s (Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ())))
M s (Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType (Bool, Maybe Sub))))
prepareBatchSubs clntServiceId_ batch = do
let (subMsgQs, rcvAssocQs, ntfAssocQs) = foldr partitionSubs ([], [], []) batch
partitionSubs t (msgQs, rcvQs, ntfQs) = case t of
(Just (q, qr), (_, _, Cmd SRecipient SUB))
| clntServiceId_ /= rcvServiceId qr -> (q : msgQs, q : rcvQs, ntfQs)
| otherwise -> (q : msgQs, rcvQs, ntfQs)
| clntServiceId_ /= rcvServiceId qr -> ((q, qr) : msgQs, q : rcvQs, ntfQs)
| otherwise -> ((q, qr) : msgQs, rcvQs, ntfQs)
(Just (q, qr), (_, _, Cmd SNotifier NSUB))
| clntServiceId_ /= (notifier qr >>= ntfServiceId) -> (msgQs, rcvQs, q : ntfQs)
_ -> (msgQs, rcvQs, ntfQs)
liftIO $ runExceptT $ do
runExceptT $ do
rcvAssocs <- ifNotNull rcvAssocQs $ setService SRecipientService clntServiceId_
ntfAssocs <- ifNotNull ntfAssocQs $ setService SNotifierService clntServiceId_
msgs <- ifNotNull subMsgQs $ tryPeekMsgs ms
pure (msgs, rcvAssocs, ntfAssocs)
subs <- lift $ M.fromList <$> mapM (\(q, qr) -> (recipientId q,) <$> subscribeQueue clntServiceId_ q qr) (filter (subscribable rcvAssocs) subMsgQs)
msgs <- ifNotNull (map fst subMsgQs) $ mapExceptT liftIO . tryPeekMsgs ms
pure (msgs, rcvAssocs, ntfAssocs, subs)
where
ifNotNull qs f = if null qs then pure M.empty else f qs
setService :: (PartyI p, ServiceParty p) => SParty p -> Maybe ServiceId -> [StoreQueue s] -> ExceptT ErrorType IO (Map RecipientId (Either ErrorType ()))
setService party sId = ExceptT . setQueueServices (queueStore ms) party sId
setService :: (PartyI p, ServiceParty p) => SParty p -> Maybe ServiceId -> [StoreQueue s] -> ExceptT ErrorType (ReaderT (Env s) IO) (Map RecipientId (Either ErrorType ()))
setService party sId = ExceptT . liftIO . setQueueServices (queueStore ms) party sId
subscribable rcvAssocs (q, QueueRec {status}) = case status of
EntityBlocked _ -> False
_ -> all isRight $ M.lookup (recipientId q) rcvAssocs
subscribeQueue :: Maybe ServiceId -> StoreQueue s -> QueueRec -> M s (Either ErrorType (Bool, Maybe Sub))
subscribeQueue clntServiceId q QueueRec {rcvServiceId} =
liftIO (TM.lookupIO rId $ subscriptions clnt) >>= \case
Nothing -> Right <$> sharedSubscribeQueue clntServiceId rId q rcvServiceId processSub subscriptions serviceSubsCount (newSubscription NoSub) rcvServices
Just s@Sub {subThread} -> do
stats <- asks serverStats
case subThread of
ProhibitSub -> do
-- cannot use SUB in the same connection where GET was used
incStat $ qSubProhibited stats
pure $ Left $ CMD PROHIBITED
_ -> do
incStat $ qSubDuplicate stats
Right (True, Just s) <$ atomically (writeTVar (delivered s) Nothing)
where
rId = recipientId q
sharedSubscribeQueue ::
Maybe ServiceId ->
QueueId ->
StoreQueue s ->
Maybe ServiceId ->
(Client s -> ClientSub -> IO ()) ->
(Client s -> TMap QueueId sub) ->
(Client s -> TVar (Int64, IdsHash)) ->
STM sub ->
(ServerStats -> ServiceStats) ->
M s (Bool, Maybe sub)
sharedSubscribeQueue clntServiceId entId q queueServiceId processSub' clientSubs clientServiceSubs mkSub servicesSel = do
stats <- asks serverStats
let incSrvStat sel = incStat $ sel $ servicesSel stats
writeSub = processSub' clnt $ CSClient entId queueServiceId clntServiceId
liftIO $ case clntServiceId of
Just serviceId
| queueServiceId == Just serviceId -> do
-- duplicate queue-service association - can only happen in case of response error/timeout
hasSub <- atomically $ ifM hasServiceSub (pure True) (False <$ incServiceQueueSubs)
unless hasSub $ do
writeSub
incSrvStat srvSubCount
incSrvStat srvSubQueues
incSrvStat srvAssocDuplicate
pure (hasSub, Nothing)
| otherwise -> do
-- association already done in prepareBatchSubs
hasSub <- atomically $ (<$ incServiceQueueSubs) =<< hasServiceSub
writeSub
unless hasSub $ incSrvStat srvSubCount
incSrvStat srvSubQueues
incSrvStat $ maybe srvAssocNew (const srvAssocUpdated) queueServiceId
pure (hasSub, Nothing)
where
hasServiceSub = ((0 /=) . fst) <$> readTVar (clientServiceSubs clnt)
-- This function is used when queue association with the service is created.
incServiceQueueSubs = modifyTVar' (clientServiceSubs clnt) $ addServiceSubs (1, queueIdHash (recipientId q)) -- service count and IDS hash
Nothing -> case queueServiceId of
Just _ -> do
-- unassociation already done in prepareBatchSubs
incSrvStat srvAssocRemoved
-- getSubscription may find a subscription for receiving service, where clientSubs also hold active deliveries for service subscriptions.
-- For notification service it can only find one if storage and session states diverge.
r <- getSubscription
writeSub
pure r
Nothing -> do
r@(hasSub, _) <- getSubscription
unless hasSub writeSub
pure r
where
getSubscription = either ((False,) . Just) ((True,) . Just) <$> TM.lookupOrInsertIO entId mkSub (clientSubs clnt)
processProxiedCmd :: Transmission (Command 'ProxiedClient) -> M s (Maybe ResponseAndMessage)
processProxiedCmd (corrId, EntityId sessId, command) = (\t -> ((corrId, EntityId sessId, t), Nothing)) <$$> case command of
@@ -1573,7 +1630,7 @@ client
mkIncProxyStats ps psOwn own sel = do
incStat $ sel ps
when own $ incStat $ sel psOwn
processCommand :: Maybe ServiceId -> Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ())) -> VerifiedTransmission s -> M s (Maybe ResponseAndMessage)
processCommand :: Maybe ServiceId -> Either ErrorType (Map RecipientId Message, Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType ()), Map RecipientId (Either ErrorType (Bool, Maybe Sub))) -> VerifiedTransmission s -> M s (Maybe ResponseAndMessage)
processCommand clntServiceId batchSubs (q_, (corrId, entId, cmd)) = case cmd of
Cmd SProxiedClient command -> processProxiedCmd (corrId, entId, command)
Cmd SSender command -> case command of
@@ -1590,7 +1647,7 @@ client
Cmd SNotifier NSUB -> response . (corrId,entId,) <$> case q_ of
Just (q, QueueRec {notifier = Just ntfCreds}) ->
either (pure . ERR) (\_ -> subscribeNotifications q ntfCreds)
$ batchSubs >>= \(_, _, ntfAssocs) -> sequence (M.lookup (recipientId q) ntfAssocs)
$ batchSubs >>= \(_, _, ntfAssocs, _) -> sequence (M.lookup (recipientId q) ntfAssocs)
_ -> pure $ ERR INTERNAL
Cmd SNotifierService (NSUBS n idsHash) -> response . (corrId,entId,) <$> case clntServiceId of
Just serviceId -> subscribeServiceNotifications serviceId (n, idsHash)
@@ -1603,9 +1660,9 @@ client
pure $ allowNewQueues && maybe True ((== auth_) . Just) newQueueBasicAuth
Cmd SRecipient command ->
case command of
SUB -> case batchSubs >>= \(msgs, rcvAssocs, _) -> sequence (M.lookup entId rcvAssocs) $> msgs of
SUB -> case batchSubs >>= \(msgs, rcvAssocs, _, subs) -> sequence (M.lookup entId rcvAssocs) $> (msgs, subs) of
Left e -> pure $ Just (err e, Nothing)
Right msgs -> withQueue' $ subscribeQueueAndDeliver $ M.lookup entId msgs
Right (msgs, subs) -> withQueue' $ deliverSubMsg (M.lookup entId msgs) (M.lookup entId subs)
GET -> withQueue getMessage
ACK msgId -> withQueue $ acknowledgeMsg msgId
KEY sKey -> withQueue $ \q _ -> either err (corrId,entId,) <$> secureQueue_ q sKey
@@ -1726,7 +1783,7 @@ client
Right nc_ -> do
incStat . ntfCreated =<< asks serverStats
forM_ nc_ $ \NtfCreds {notifierId = nId, ntfServiceId} ->
atomically $ writeTQueue (subQ ntfSubscribers) (CSDeleted nId ntfServiceId, clientId)
liftIO $ processNtfSub clnt $ CSDeleted nId ntfServiceId
pure $ NID notifierId rcvPublicDhKey
deleteQueueNotifier_ :: StoreQueue s -> M s (Transmission BrokerMsg)
@@ -1737,7 +1794,7 @@ client
stats <- asks serverStats
deleted <- asks ntfStore >>= liftIO . (`deleteNtfs` nId)
when (deleted > 0) $ liftIO $ atomicModifyIORef'_ (ntfCount stats) (subtract deleted)
atomically $ writeTQueue (subQ ntfSubscribers) (CSDeleted nId ntfServiceId, clientId)
liftIO $ processNtfSub clnt $ CSDeleted nId ntfServiceId
incStat $ ntfDeleted stats
pure ok
Right Nothing -> pure ok
@@ -1746,32 +1803,24 @@ client
suspendQueue_ :: (StoreQueue s, QueueRec) -> M s (Transmission BrokerMsg)
suspendQueue_ (q, _) = liftIO $ either err (const ok) <$> suspendQueue (queueStore ms) q
subscribeQueueAndDeliver :: Maybe Message -> StoreQueue s -> QueueRec -> M s ResponseAndMessage
subscribeQueueAndDeliver msg_ q qr@QueueRec {rcvServiceId} =
liftIO (TM.lookupIO entId $ subscriptions clnt) >>= \case
Nothing ->
deliver =<< sharedSubscribeQueue q rcvServiceId subscribers subscriptions serviceSubsCount (newSubscription NoSub) rcvServices
Just s@Sub {subThread} -> do
stats <- asks serverStats
case subThread of
ProhibitSub -> do
-- cannot use SUB in the same connection where GET was used
incStat $ qSubProhibited stats
pure (err (CMD PROHIBITED), Nothing)
_ -> do
incStat $ qSubDuplicate stats
atomically (writeTVar (delivered s) Nothing) >> deliver (True, Just s)
deliverSubMsg :: Maybe Message -> Maybe (Either ErrorType (Bool, Maybe Sub)) -> StoreQueue s -> QueueRec -> M s ResponseAndMessage
deliverSubMsg msg_ r_ _ qr = case r_ of
Just (Right r) -> deliver r
Just (Left e) -> pure (err e, Nothing)
Nothing -> pure (err INTERNAL, Nothing)
where
deliver :: (Bool, Maybe Sub) -> M s ResponseAndMessage
deliver (hasSub, sub_) = do
stats <- asks serverStats
fmap (either ((,Nothing) . err) id) $ liftIO $ runExceptT $ do
msg' <- forM msg_ $ \msg -> liftIO $ do
msg' <- fmap join $ forM msg_ $ \msg -> liftIO $ do
ts <- getSystemSeconds
sub <- maybe getSub pure sub_
atomically $ setDelivered sub msg ts
unless hasSub $ incStat $ qSub stats
pure (NoCorrId, entId, MSG (encryptMsg qr msg))
ifM
(atomically $ trySetDelivered sub msg ts)
(pure $ Just (NoCorrId, entId, MSG (encryptMsg qr msg)))
(pure Nothing)
pure ((corrId, entId, SOK clntServiceId), msg')
getSub :: IO Sub
@@ -1784,7 +1833,7 @@ client
Nothing -> do
sub <- atomically $ newSubscription NoSub
atomically $ TM.insert rId sub $ subscriptions clnt
atomically $ writeTQueue (subQ subscribers) (CSClient rId rcvServiceId rcvServiceId, clientId)
liftIO $ processSub clnt $ CSClient rId rcvServiceId rcvServiceId
-- clients that use GET are not added to server subscribers
getMessage :: StoreQueue s -> QueueRec -> M s (Transmission BrokerMsg)
@@ -1840,66 +1889,14 @@ client
subscribeNotifications :: StoreQueue s -> NtfCreds -> M s BrokerMsg
subscribeNotifications q NtfCreds {ntfServiceId} = do
(hasSub, _) <- sharedSubscribeQueue q ntfServiceId ntfSubscribers ntfSubscriptions ntfServiceSubsCount (pure ()) ntfServices
(hasSub, _) <- sharedSubscribeQueue clntServiceId entId q ntfServiceId processNtfSub ntfSubscriptions ntfServiceSubsCount (pure ()) ntfServices
when (isNothing clntServiceId) $
asks serverStats >>= incStat . (if hasSub then ntfSubDuplicate else ntfSub)
pure $ SOK clntServiceId
sharedSubscribeQueue ::
StoreQueue s ->
Maybe ServiceId ->
ServerSubscribers s ->
(Client s -> TMap QueueId sub) ->
(Client s -> TVar (Int64, IdsHash)) ->
STM sub ->
(ServerStats -> ServiceStats) ->
M s (Bool, Maybe sub)
sharedSubscribeQueue q queueServiceId srvSubscribers clientSubs clientServiceSubs mkSub servicesSel = do
stats <- asks serverStats
let incSrvStat sel = incStat $ sel $ servicesSel stats
writeSub = writeTQueue (subQ srvSubscribers) (CSClient entId queueServiceId clntServiceId, clientId)
liftIO $ case clntServiceId of
Just serviceId
| queueServiceId == Just serviceId -> do
-- duplicate queue-service association - can only happen in case of response error/timeout
hasSub <- atomically $ ifM hasServiceSub (pure True) (False <$ incServiceQueueSubs)
unless hasSub $ do
atomically writeSub
incSrvStat srvSubCount
incSrvStat srvSubQueues
incSrvStat srvAssocDuplicate
pure (hasSub, Nothing)
| otherwise -> do
-- association already done in prepareBatchSubs
hasSub <- atomically $ (<$ incServiceQueueSubs) =<< hasServiceSub
atomically writeSub
unless hasSub $ incSrvStat srvSubCount
incSrvStat srvSubQueues
incSrvStat $ maybe srvAssocNew (const srvAssocUpdated) queueServiceId
pure (hasSub, Nothing)
where
hasServiceSub = ((0 /=) . fst) <$> readTVar (clientServiceSubs clnt)
-- This function is used when queue association with the service is created.
incServiceQueueSubs = modifyTVar' (clientServiceSubs clnt) $ addServiceSubs (1, queueIdHash (recipientId q)) -- service count and IDS hash
Nothing -> case queueServiceId of
Just _ -> do
-- unassociation already done in prepareBatchSubs
incSrvStat srvAssocRemoved
-- getSubscription may find a subscription for receiving service, where clientSubs also hold active deliveries for service subscriptions.
-- For notification service it can only find one if storage and session states diverge.
r <- getSubscription
atomically writeSub
pure r
Nothing -> do
r@(hasSub, _) <- getSubscription
unless hasSub $ atomically writeSub
pure r
where
getSubscription = either ((False,) . Just) ((True,) . Just) <$> TM.lookupOrInsertIO entId mkSub (clientSubs clnt)
subscribeServiceMessages :: ServiceId -> (Int64, IdsHash) -> M s BrokerMsg
subscribeServiceMessages serviceId expected =
sharedSubscribeService SRecipientService serviceId expected subscribers serviceSubscribed serviceSubsCount rcvServices >>= \case
sharedSubscribeService SRecipientService serviceId expected processSub serviceSubscribed serviceSubsCount rcvServices >>= \case
Left e -> pure $ ERR e
Right (hasSub, (count, idsHash)) -> do
stats <- asks serverStats
@@ -1930,17 +1927,18 @@ client
Right _sub -> pure (qCnt + 1, msgCnt, dupCnt + 1, evts)
Left sub -> do
ts <- getSystemSeconds
atomically $ setDelivered sub msg ts
atomically $ writeTBQueue msgQ [(NoCorrId, rId, MSG (encryptMsg qr msg))]
pure (qCnt + 1, msgCnt + 1, dupCnt, evts)
ifM
(atomically $ trySetDelivered sub msg ts)
((qCnt + 1, msgCnt + 1, dupCnt, evts) <$ atomically (writeTBQueue msgQ [(NoCorrId, rId, MSG (encryptMsg qr msg))]))
(pure (qCnt + 1, msgCnt, dupCnt + 1, evts))
getSubscription rId = TM.lookupOrInsertIO rId (newSubscription NoSub) (subscriptions clnt)
subscribeServiceNotifications :: ServiceId -> (Int64, IdsHash) -> M s BrokerMsg
subscribeServiceNotifications serviceId expected =
either ERR (uncurry SOKS . snd) <$> sharedSubscribeService SNotifierService serviceId expected ntfSubscribers ntfServiceSubscribed ntfServiceSubsCount ntfServices
either ERR (uncurry SOKS . snd) <$> sharedSubscribeService SNotifierService serviceId expected processNtfSub ntfServiceSubscribed ntfServiceSubsCount ntfServices
sharedSubscribeService :: (PartyI p, ServiceParty p) => SParty p -> ServiceId -> (Int64, IdsHash) -> ServerSubscribers s -> (Client s -> TVar Bool) -> (Client s -> TVar (Int64, IdsHash)) -> (ServerStats -> ServiceStats) -> M s (Either ErrorType (Bool, (Int64, IdsHash)))
sharedSubscribeService party serviceId (count, idsHash) srvSubscribers clientServiceSubscribed clientServiceSubs servicesSel = do
sharedSubscribeService :: (PartyI p, ServiceParty p) => SParty p -> ServiceId -> (Int64, IdsHash) -> (Client s -> ClientSub -> IO ()) -> (Client s -> TVar Bool) -> (Client s -> TVar (Int64, IdsHash)) -> (ServerStats -> ServiceStats) -> M s (Either ErrorType (Bool, (Int64, IdsHash)))
sharedSubscribeService party serviceId (count, idsHash) processSub' clientServiceSubscribed clientServiceSubs servicesSel = do
subscribed <- readTVarIO $ clientServiceSubscribed clnt
stats <- asks serverStats
liftIO $ runExceptT $
@@ -1960,7 +1958,7 @@ client
| diff > 0 -> incSrvStat srvSubMore 1 >> incSrvStat srvSubMoreTotal diff
| diff < 0 -> incSrvStat srvSubFewer 1 >> incSrvStat srvSubFewerTotal (- diff)
| otherwise -> incSrvStat srvSubDiff 1
atomically $ writeTQueue (subQ srvSubscribers) (CSService serviceId subsChange, clientId)
liftIO $ processSub' clnt $ CSService serviceId subsChange
pure (count', idsHash')
acknowledgeMsg :: MsgId -> StoreQueue s -> QueueRec -> M s (Transmission BrokerMsg)
@@ -2204,7 +2202,7 @@ client
resolveNameMsg (thVersion clntTHParams) nenv d >>= sendResolved
-- INTERNAL because processCommand never returns Nothing for sender commands;
-- `fst` drops the empty message only returned for SUB.
_ -> Just . maybe (corrId', entId', ERR INTERNAL) fst <$> lift (processCommand Nothing (Right (M.empty, M.empty, M.empty)) t'')
_ -> Just . maybe (corrId', entId', ERR INTERNAL) fst <$> lift (processCommand Nothing (Right (M.empty, M.empty, M.empty, M.empty)) t'')
stats <- asks serverStats
incStat $ pMsgFwdsRecv stats
traverse encodeResp r_ >>= \case
@@ -2246,6 +2244,9 @@ client
let !msgId = messageId msg
writeTVar delivered $ Just (msgId, ts)
trySetDelivered :: Sub -> Message -> SystemSeconds -> STM Bool
trySetDelivered sub msg ts = readTVar (delivered sub) >>= maybe (True <$ setDelivered sub msg ts) (const $ pure False)
delQueueAndMsgs :: (StoreQueue s, QueueRec) -> M s (Transmission BrokerMsg)
delQueueAndMsgs (q, QueueRec {rcvServiceId}) = do
liftIO (deleteQueue ms q) >>= \case
@@ -2256,14 +2257,14 @@ client
sub <- atomically $ TM.lookupDelete entId $ subscriptions clnt
liftIO $ mapM_ cancelSub sub
when (isJust rcvServiceId) $ atomically $ modifyTVar' (serviceSubsCount clnt) $ subtractServiceSubs (1, queueIdHash (recipientId q))
atomically $ writeTQueue (subQ subscribers) (CSDeleted entId rcvServiceId, clientId)
liftIO $ processSub clnt $ CSDeleted entId rcvServiceId
forM_ (notifier qr) $ \NtfCreds {notifierId = nId, ntfServiceId} -> do
-- queue is deleted by a different client from the one subscribed to notifications,
-- so we don't need to remove subscription from the current client.
stats <- asks serverStats
deleted <- asks ntfStore >>= liftIO . (`deleteNtfs` nId)
when (deleted > 0) $ liftIO $ atomicModifyIORef'_ (ntfCount stats) (subtract deleted)
atomically $ writeTQueue (subQ ntfSubscribers) (CSDeleted nId ntfServiceId, clientId)
liftIO $ processNtfSub clnt $ CSDeleted nId ntfServiceId
updateDeletedStats qr
pure ok
Left e -> pure $ err e
+2 -4
View File
@@ -347,8 +347,7 @@ data Server s = Server
newtype ServerClients s = ServerClients {serverClients :: TVar (IntMap (Client s))}
data ServerSubscribers s = ServerSubscribers
{ subQ :: TQueue (ClientSub, ClientId),
queueSubscribers :: SubscribedClients s,
{ queueSubscribers :: SubscribedClients s,
serviceSubscribers :: SubscribedClients s, -- service clients with long-term certificates that have subscriptions
totalServiceSubs :: TVar (Int64, IdsHash),
subClients :: TVar IntSet, -- clients with individual or service subscriptions
@@ -486,13 +485,12 @@ deleteServerClient cId Server {clients} = atomically $ modifyTVar' (serverClient
newServerSubscribers :: IO (ServerSubscribers s)
newServerSubscribers = do
subQ <- newTQueueIO
queueSubscribers <- SubscribedClients <$> TM.emptyIO
serviceSubscribers <- SubscribedClients <$> TM.emptyIO
totalServiceSubs <- newTVarIO (0, mempty)
subClients <- newTVarIO IS.empty
pendingEvents <- newTVarIO IM.empty
pure ServerSubscribers {subQ, queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents}
pure ServerSubscribers {queueSubscribers, serviceSubscribers, totalServiceSubs, subClients, pendingEvents}
newClient :: ClientId -> Natural -> THandleParams SMPVersion 'TServer -> SystemTime -> IO (Client s)
newClient clientId qSize clientTHParams createdAt = do
+93 -3
View File
@@ -92,11 +92,15 @@ serverTests = do
describe "Concurrent sending and delivery" testConcurrentSendDelivery
describe "SUB while delivery waits" testSubPendingDelivery
describe "SUB to a queue blocked by the operator" testSubBlockedQueue
describe "SUB and SEND in one block" testSubSendBlock
describe "switch subscription with SUB and SEND in one block" testSwitchSubSendBlock
describe "two SUBs in one block" testSubTwiceBlock
describe "subscribe the same queues from two connections at once" testSubSameQueuesTwoConnections
describe "Service message subscriptions" $ do
testServiceDeliverSubscribe
testServiceUpgradeAndDowngrade
testServiceSubsTotalCount
testServiceSubSendBlock
testServiceSubAssociatedQueue
testServiceDowngradeConnected
testServiceSubsReplacesConnection
@@ -721,8 +725,8 @@ testSubPendingDelivery =
Resp "1" _ OK <- sendRecv sh ("", "1", sId, _SEND "hello")
signSend_ rh rKey Nothing ("2", rId, SUB)
threadDelay 500000
let drain st@(n, _, mId_) = case mId_ of
Just mId | n == length fillers -> pure mId
let drain st@(n, subscribed, mId_) = case mId_ of
Just mId | n == length fillers && subscribed -> pure mId
_ -> 5000000 `timeout` tGetClient rh >>= maybe (error "drain timeout") (foldM received st) >>= drain
received st@(n, subscribed, mId_) = \case
Resp "" _ (LNK _ _) -> pure st
@@ -731,7 +735,7 @@ testSubPendingDelivery =
Resp "" rId' (Msg mId msg)
| rId' == rId -> do
dec mId msg `shouldBe` Right "hello"
pure (n, subscribed, if subscribed then Just mId else mId_)
pure (n, subscribed, Just mId)
| rId' `elem` map fst fillers -> pure (n + 1, subscribed, mId_)
r -> error $ "unexpected event: " <> show r
mId <- drain (0 :: Int, False, Nothing)
@@ -753,6 +757,67 @@ testSubPendingDelivery =
_ -> pure Nothing
]
testSubSendBlock :: SpecWith (ASrvTransport, AStoreType)
testSubSendBlock =
it "should deliver message sent in the same block as SUB" $ \(ATransport t, msType) ->
smpTest t msType $ \h@THandle {params} -> do
g <- C.newRandom
(rPub, rKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g
(dhPub, dhPriv :: C.PrivateKeyX25519) <- atomically $ C.generateKeyPair g
Resp "1" NoEntity (Ids rId sId srvDh) <- signSendRecv h rKey ("1", NoEntity, NEW (NewQueueReq rPub dhPub Nothing SMOnlyCreate (Just (QRMessaging Nothing)) Nothing))
let dec = decryptMsgV3 $ C.dh' srvDh dhPriv
TransmissionForAuth {tForAuth, tToSend = subT} = encodeTransmissionForAuth params (CorrId "2", rId, SUB)
TransmissionForAuth {tToSend = sendT} = encodeTransmissionForAuth params (CorrId "3", sId, _SEND "hello")
Right subAuth <- pure $ authTransmission (thAuth params) False (Just rKey) Nothing tForAuth
[Right (), Right ()] <- tPut h [Right (subAuth, subT), Right (Nothing, sendT)]
Resp "" _ (Msg mId msg) <- tGet1 h
dec mId msg `shouldBe` Right "hello"
[Resp "2" _ (SOK Nothing), Resp "3" _ OK] <- tGetClient h
Resp "4" _ OK <- signSendRecv h rKey ("4", rId, ACK mId)
pure ()
testSwitchSubSendBlock :: SpecWith (ASrvTransport, AStoreType)
testSwitchSubSendBlock =
it "should deliver message sent in the same block as SUB to the new connection" $ \(ATransport t, msType) ->
smpTest2 t msType $ \rh1 rh2@THandle {params} -> do
g <- C.newRandom
(rPub, rKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g
(dhPub, dhPriv :: C.PrivateKeyX25519) <- atomically $ C.generateKeyPair g
Resp "1" NoEntity (Ids rId sId srvDh) <- signSendRecv rh1 rKey ("1", NoEntity, New rPub dhPub)
let dec = decryptMsgV3 $ C.dh' srvDh dhPriv
TransmissionForAuth {tForAuth, tToSend = subT} = encodeTransmissionForAuth params (CorrId "2", rId, SUB)
TransmissionForAuth {tToSend = sendT} = encodeTransmissionForAuth params (CorrId "3", sId, _SEND "hello")
Right subAuth <- pure $ authTransmission (thAuth params) False (Just rKey) Nothing tForAuth
[Right (), Right ()] <- tPut rh2 [Right (subAuth, subT), Right (Nothing, sendT)]
Resp "" _ end <- tGet1 rh1
end `shouldBe` END
Resp "" _ (Msg mId msg) <- tGet1 rh2
dec mId msg `shouldBe` Right "hello"
[Resp "2" _ (SOK Nothing), Resp "3" _ OK] <- tGetClient rh2
Resp "4" _ OK <- signSendRecv rh2 rKey ("4", rId, ACK mId)
pure ()
testSubTwiceBlock :: SpecWith (ASrvTransport, AStoreType)
testSubTwiceBlock =
it "should deliver message once for two SUBs in the same block" $ \(ATransport t, msType) ->
smpTest t msType $ \h@THandle {params} -> do
g <- C.newRandom
(rPub, rKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g
(dhPub, dhPriv :: C.PrivateKeyX25519) <- atomically $ C.generateKeyPair g
Resp "1" NoEntity (Ids rId sId srvDh) <- signSendRecv h rKey ("1", NoEntity, NEW (NewQueueReq rPub dhPub Nothing SMOnlyCreate (Just (QRMessaging Nothing)) Nothing))
let dec = decryptMsgV3 $ C.dh' srvDh dhPriv
Resp "2" _ OK <- sendRecv h ("", "2", sId, _SEND "hello")
let TransmissionForAuth {tForAuth = forAuth1, tToSend = sub1} = encodeTransmissionForAuth params (CorrId "3", rId, SUB)
TransmissionForAuth {tForAuth = forAuth2, tToSend = sub2} = encodeTransmissionForAuth params (CorrId "4", rId, SUB)
Right auth1 <- pure $ authTransmission (thAuth params) False (Just rKey) Nothing forAuth1
Right auth2 <- pure $ authTransmission (thAuth params) False (Just rKey) Nothing forAuth2
[Right (), Right ()] <- tPut h [Right (auth1, sub1), Right (auth2, sub2)]
[Resp "3" _ (SOK Nothing), Resp "4" _ (SOK Nothing), Resp "" _ (Msg mId msg)] <- tGetClient h
dec mId msg `shouldBe` Right "hello"
Nothing <- 100000 `timeout` tGetClient @SMPVersion @ErrorType @BrokerMsg h
Resp "5" _ OK <- signSendRecv h rKey ("5", rId, ACK mId)
pure ()
testSubBlockedQueue :: SpecWith (ASrvTransport, AStoreType)
testSubBlockedQueue =
it "should return BLOCKED to SUB after the operator blocks the queue" $ \(ATransport t, msType) -> do
@@ -1084,6 +1149,31 @@ testServiceSubsTotalCount =
threadDelay 1500000
B.readFile testPrometheusMetricsFile >>= \m -> readServiceSubsMetric (B.unpack m) `shouldBe` Just 3
testServiceSubSendBlock :: SpecWith (ASrvTransport, AStoreType)
testServiceSubSendBlock =
it "should deliver message sent in the same block as service SUB" $ \(at@(ATransport t), msType) -> do
g <- C.newRandom
creds <- genCredentials g Nothing (0, 2400) "localhost"
let (_fp, tlsCred) = tlsCredentials [creds]
serviceKeys@(_, servicePK) <- atomically $ C.generateKeyPair g
withSmpServerConfigOn at (cfgMS msType) testPort $ \_ -> do
(rPub, rKey) <- atomically $ C.generateAuthKeyPair C.SEd25519 g
(dhPub, dhPriv :: C.PrivateKeyX25519) <- atomically $ C.generateKeyPair g
(rId, sId, dec, serviceId) <- runSMPServiceClient t (tlsCred, serviceKeys) $ \sh -> do
Resp "1" NoEntity (Ids_ rId sId srvDh serviceId) <- serviceSignSendRecv sh rKey servicePK ("1", NoEntity, New rPub dhPub)
pure (rId, sId, decryptMsgV3 $ C.dh' srvDh dhPriv, serviceId)
runSMPServiceClient t (tlsCred, serviceKeys) $ \sh@THandle {params} -> do
let TransmissionForAuth {tForAuth, tToSend = subT} = encodeTransmissionForAuth params (CorrId "2", rId, SUB)
TransmissionForAuth {tToSend = sendT} = encodeTransmissionForAuth params (CorrId "3", sId, _SEND "hello")
Right subAuth <- pure $ authTransmission (thAuth params) True (Just rKey) Nothing tForAuth
[Right (), Right ()] <- tPut sh [Right (subAuth, subT), Right (Nothing, sendT)]
Resp "" _ (Msg mId msg) <- tGet1 sh
dec mId msg `shouldBe` Right "hello"
[Resp "2" _ (SOK (Just serviceId')), Resp "3" _ OK] <- tGetClient sh
serviceId' `shouldBe` serviceId
Resp "4" _ OK <- signSendRecv sh rKey ("4", rId, ACK mId)
pure ()
testServiceSubAssociatedQueue :: SpecWith (ASrvTransport, AStoreType)
testServiceSubAssociatedQueue =
it "should deliver the waiting message when the service subscribes its queue with SUB after reconnecting" $ \(at@(ATransport t), msType) -> do