From cff7cc67797d3b2a7ab03c6bf2b726d5725f26cd Mon Sep 17 00:00:00 2001 From: Jan Dittberner Date: Sun, 28 Oct 2018 19:22:32 +0100 Subject: [PATCH] Start documenting CommModule - add a file meant to collect general observations - add a file meant to collect information related to the database schema - add a glossary file - add documentation for the CommModule files in source/directories.rst - start signer protocol specification in source/signer.rst - add support for block and sequence diagrams via sphinxcontrib-blockdiag and sphinxcontrib-seqdiag --- Pipfile | 3 + Pipfile.lock | 88 +++++++++++++++- source/conf.py | 3 + source/directories.rst | 167 +++++++++++++++++++++++++++++++ source/general.rst | 26 +++++ source/glossary.rst | 16 +++ source/index.rst | 4 +- source/signer.rst | 222 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 527 insertions(+), 2 deletions(-) create mode 100644 source/general.rst create mode 100644 source/glossary.rst create mode 100644 source/signer.rst diff --git a/Pipfile b/Pipfile index c9e13e7..91155e2 100644 --- a/Pipfile +++ b/Pipfile @@ -8,6 +8,9 @@ sphinx = "*" GitPython = "*" certifi = "*" requests = "*" +sphinxcontrib-phpdomain = "*" +sphinxcontrib-blockdiag = "*" +sphinxcontrib-seqdiag = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index eca8acd..534cb7a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "1666cbe0230e5956dfa4b61f4811218f730e7937181a37ab44b32f0270d3bd11" + "sha256": "49436bd593c2b93213655f26a631e356f4630a0358f6515516588831ff8ee25f" }, "pipfile-spec": 6, "requires": { @@ -30,6 +30,13 @@ ], "version": "==2.6.0" }, + "blockdiag": { + "hashes": [ + "sha256:8dd6570a2ac41b3c0dfe5706de20913cdbebe1bbd2e6dea9ebc13db79df8c151", + "sha256:929125db1cb59145e09dc561021389c7ca71108ef4e4c51a12728eea5b75fccc" + ], + "version": "==1.5.4" + }, "certifi": { "hashes": [ "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c", @@ -53,6 +60,12 @@ ], "version": "==0.14" }, + "funcparserlib": { + "hashes": [ + "sha256:b7992eac1a3eb97b3d91faa342bfda0729e990bd8a43774c1592c091e563c91d" + ], + "version": "==0.3.6" + }, "gitdb2": { "hashes": [ "sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2", @@ -102,6 +115,41 @@ ], "version": "==18.0" }, + "pillow": { + "hashes": [ + "sha256:00203f406818c3f45d47bb8fe7e67d3feddb8dcbbd45a289a1de7dd789226360", + "sha256:0616f800f348664e694dddb0b0c88d26761dd5e9f34e1ed7b7a7d2da14b40cb7", + "sha256:1f7908aab90c92ad85af9d2fec5fc79456a89b3adcc26314d2cde0e238bd789e", + "sha256:2ea3517cd5779843de8a759c2349a3cd8d3893e03ab47053b66d5ec6f8bc4f93", + "sha256:48a9f0538c91fc136b3a576bee0e7cd174773dc9920b310c21dcb5519722e82c", + "sha256:5280ebc42641a1283b7b1f2c20e5b936692198b9dd9995527c18b794850be1a8", + "sha256:5e34e4b5764af65551647f5cc67cf5198c1d05621781d5173b342e5e55bf023b", + "sha256:63b120421ab85cad909792583f83b6ca3584610c2fe70751e23f606a3c2e87f0", + "sha256:696b5e0109fe368d0057f484e2e91717b49a03f1e310f857f133a4acec9f91dd", + "sha256:870ed021a42b1b02b5fe4a739ea735f671a84128c0a666c705db2cb9abd528eb", + "sha256:916da1c19e4012d06a372127d7140dae894806fad67ef44330e5600d77833581", + "sha256:9303a289fa0811e1c6abd9ddebfc770556d7c3311cb2b32eff72164ddc49bc64", + "sha256:9577888ecc0ad7d06c3746afaba339c94d62b59da16f7a5d1cff9e491f23dace", + "sha256:987e1c94a33c93d9b209315bfda9faa54b8edfce6438a1e93ae866ba20de5956", + "sha256:99a3bbdbb844f4fb5d6dd59fac836a40749781c1fa63c563bc216c27aef63f60", + "sha256:99db8dc3097ceafbcff9cb2bff384b974795edeb11d167d391a02c7bfeeb6e16", + "sha256:a5a96cf49eb580756a44ecf12949e52f211e20bffbf5a95760ac14b1e499cd37", + "sha256:aa6ca3eb56704cdc0d876fc6047ffd5ee960caad52452fbee0f99908a141a0ae", + "sha256:aade5e66795c94e4a2b2624affeea8979648d1b0ae3fcee17e74e2c647fc4a8a", + "sha256:b78905860336c1d292409e3df6ad39cc1f1c7f0964e66844bbc2ebfca434d073", + "sha256:b92f521cdc4e4a3041cc343625b699f20b0b5f976793fb45681aac1efda565f8", + "sha256:bfde84bbd6ae5f782206d454b67b7ee8f7f818c29b99fd02bf022fd33bab14cb", + "sha256:c2b62d3df80e694c0e4a0ed47754c9480521e25642251b3ab1dff050a4e60409", + "sha256:c5e2be6c263b64f6f7656e23e18a4a9980cffc671442795682e8c4e4f815dd9f", + "sha256:c99aa3c63104e0818ec566f8ff3942fb7c7a8f35f9912cb63fd8e12318b214b2", + "sha256:dae06620d3978da346375ebf88b9e2dd7d151335ba668c995aea9ed07af7add4", + "sha256:db5499d0710823fa4fb88206050d46544e8f0e0136a9a5f5570b026584c8fd74", + "sha256:f36baafd82119c4a114b9518202f2a983819101dcc14b26e43fc12cbefdce00e", + "sha256:f52b79c8796d81391ab295b04e520bda6feed54d54931708872e8f9ae9db0ea1", + "sha256:ff8cff01582fa1a7e533cb97f628531c4014af4b5f38e33cdcfe5eec29b6d888" + ], + "version": "==5.3.0" + }, "pygments": { "hashes": [ "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d", @@ -131,6 +179,13 @@ "index": "pypi", "version": "==2.20.0" }, + "seqdiag": { + "hashes": [ + "sha256:78104e7644c1a4d3a5cacb68de6a7f720793f08dd78561ef0e9e80bed63702bf", + "sha256:887cf56b00bd2492e17ef3a16c4270ff263df3c249eddea85844bb61b594785a" + ], + "version": "==0.9.6" + }, "six": { "hashes": [ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", @@ -160,6 +215,30 @@ "index": "pypi", "version": "==1.8.1" }, + "sphinxcontrib-blockdiag": { + "hashes": [ + "sha256:2d2ccde16bafb061ae8d2008f9524726e8ccd2a8502651b76a1e7f07a4ffd8eb", + "sha256:7cdff966d8f372b9536374954314a6cf4280e0e48bc2321a4f25cc7f2114f8f0" + ], + "index": "pypi", + "version": "==1.5.5" + }, + "sphinxcontrib-phpdomain": { + "hashes": [ + "sha256:14ecb0b477dddf8ce2c69b72ab15e3455a591d077264808a4f0d35f488b54a5f", + "sha256:ec0286d66233839994a2c812345bbd3f02feca28da941b552bce7d48eb8980f4" + ], + "index": "pypi", + "version": "==0.4.1" + }, + "sphinxcontrib-seqdiag": { + "hashes": [ + "sha256:83c3fdac7e083c5b217f65359c03b75af753209028db6b261b196aff19e7003f", + "sha256:c83f2b552e8e0829dbee22a13c5025f33c0b31a7e87bb589611928c2883d3db5" + ], + "index": "pypi", + "version": "==0.8.5" + }, "sphinxcontrib-websupport": { "hashes": [ "sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd", @@ -173,6 +252,13 @@ "sha256:8819bba37a02d143296a4d032373c4dd4aca11f6d4c9973335ca75f9c8475f59" ], "version": "==1.24" + }, + "webcolors": { + "hashes": [ + "sha256:030562f624467a9901f0b455fef05486a88cfb5daa1e356bd4aacea043850b59", + "sha256:b3b88e5ef2b35fa9e01e3fabe99dddf49da074459c44774c59f3ccab3be4f121" + ], + "version": "==1.8.1" } }, "develop": {} diff --git a/source/conf.py b/source/conf.py index 266ca3e..5ebc900 100644 --- a/source/conf.py +++ b/source/conf.py @@ -62,6 +62,9 @@ extensions = [ 'sphinx.ext.extlinks', 'sphinx.ext.todo', 'sphinx.ext.ifconfig', + 'sphinxcontrib.phpdomain', + 'sphinxcontrib.blockdiag', + 'sphinxcontrib.seqdiag', ] # Add any paths that contain templates here, relative to this directory. diff --git a/source/directories.rst b/source/directories.rst index fd67962..cce7201 100644 --- a/source/directories.rst +++ b/source/directories.rst @@ -14,11 +14,15 @@ The root directory contains .. _GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0 +.. index:: cgi-bin + Directory :file:`cgi-bin` ========================= The `cgi-bin` directory contains +.. index:: php + .. _cgi-bin-siteseal-cgi: - :file:`siteseal.cgi` a PHP CGI script that generates some JavaScript code @@ -28,6 +32,169 @@ The `cgi-bin` directory contains .. todo: check whether this is linked anywhere or can be removed +.. index:: commmodule +.. index:: Perl +.. index:: bash + +Directory :file:`CommModule` +============================ + +This directory contains the CommModule that is implemented in Perl: + +.. _commmodule-client-pl: + +- :file:`client.pl` the real client, running on the webserver + + The style of the Perl code seems a bit inconsistent (mix of uppercase and + lowercase function names, usage of brackets). The code uses database polling + in a loop. It might be a better idea to use some kind of queueing (Redis, + AMQP, ...) to not waste resources when there is nothing to do). Function + parameters are not named which makes the code hard to read. + + The script calls several system binaries that need to be present in + compatible versions: + + - :program:`openssl` + - :program:`xdelta` + + The script uses several Perl standard library modules as well as the + following third party modules: + + .. index:: Perl, thirdparty + + - `DBD::mysql `_ + - `DBI `_ + - `Device::SerialPort `_ + - `File::CounterFile `_ + + The script references several openssl configuration files in the HandleCerts + function that are not included in the code repository. There are some + openssl configuration files with similar names in + https://svn.cacert.org/CAcert/SystemAdministration/signer/ + + The database password is parsed from + :ref:`includes/mysql.php ` and relies on the + exact code that is defined there. Database name, user and host are hardcoded + in the DBI->connect call. + + The script implements the client side of the signer protocol which is + specified in :doc:`signer`. + + The script performs the following operations: + + - parse password from :file:`includes/mysql.php` + - read a list of CRL files and logs their SHA-1 hashes + - read :file:`serial.conf`, create a Device::SerialPort instance `$portObj`, + sets serial parameters and saves :file:`serial.conf` + - run a main loop as long as a file :file:`./client.pl-active` is present. + The main loop performs the following tasks + + - handle pending OpenPGP key signing request via ``HandleGPG()`` + - handle pending certificate signing requests: + + - personal client certificates via ``HandleCerts(0, 0)`` + - personal server certificates via ``HandleCerts(0, 1)`` + - organization client certificates via ``HandleCerts(1, 0)`` + - organization server certificates via ``HandleCerts(1, 1)`` + + - handle pending certificate revocation requests + + - personal client certificates via ``RevokeCerts(0, 0)`` + - personal server certificates via ``RevokeCerts(0, 1)`` + - organization client certificates via ``RevokeCerts(1, 0)`` + - organization server certificates via ``RevokeCerts(1, 1)`` + + - refresh :term:`CRLs ` via ``RefreshCRLs()`` in every 100st + iteration + - send a :ref:`NUL request ` to keep the signer + connection alive + - sleep for 2.7 seconds + + There is potential for optimization in the main loop. The CRL update could + be performed if a certificate has been revoked. The NUL request needs only + to be sent if no other request has been sent. + + .. todo:: describe more in-depth what each of the main loop steps does + +- :file:`commdaemon` a script to run :ref:`client.pl ` + or :ref:`server.pl ` + + This bash script is automatically restarting the :file:`{script}` given as + the first parameter as long as a file :file:`{script}-active` exists. + Informational messages and errors are logged to syslog via + :command:`logger`. + + The script is most probably used to recover from crashed scripts. This + could be implemented via :command:`supervisor` or :command:`systemd` + instead of a custom script. + +- :file:`commmodule` a script for startup/shutdown of CommModule from + /etc/init.d +- :file:`logclean.sh` maintenance script for logfiles generated by CommModule +- :file:`serial.conf` serial port configuration file + +.. _commmodule-server-pl: + +- :file:`server.pl` the real server, running on the signing server + + This script implements the signer (server) side of the signer protocol and + performs the actual signing operations. + + The script contains a some code that is duplicated by + :ref:`client.pl `. + +- :file:`usbclient.pl` obsoleted USB version of + :ref:`client.pl ` above + +.. todo: remove unused file (usbclient.pl) + +.. todo: add a serial.conf template and move the actual serial.conf into + configuration management + +.. todo: clarify why log rotation is implemented with a custom + logclean.sh script instead of using logrotate + +Directory :file:`includes` +============================== + +.. _includes-mysql-php: +.. _includes-mysql-php-sample: + +- :file:`mysql.php.sample` is a template for the database connection handling + code that is meant to be copied to :file:`mysql.php`. + + The template defines the MySQL connection as a session variable `mconn` and + tries to connect to that database. It also defines the session variables + `normalhostname`, `securehostname` and `tverify`. + + The template defines a function :php:func:`sendmail` for sending mails. + + .. php:function:: sendmail($to, $subject, $message, $from, $replyto="", \ + $toname="", $fromname="", $errorsto="returns@cacert.org", \ + $use_utf8=true) + + Send an email. The function reimplements functionality that is readily + available in PHP. The function does not properly escape headers and + sends raw SMTP commands. + + :param string $to: recipient email address + :param string $subject: subject + :param string $message: email body + :param string $from: from email address + :param string $replyto: reply-to email address + :param string $fromname: unused in the code + :param string $toname: unused in the code + :param string $errorsto: email address used for Sender and Errors-To + headers + :param bool $use_utf8: decides whether the Content-Type header uses + a charset parameter of utf-8 or iso-8859-1 + + Configuration and actual code are mixed. It would be better to have a + separate file that just includes configuration. + + This file is parsed by :ref:`CommModule/client.pl ` + format changes might break the CommModule code. + Directory :file:`www` ===================== diff --git a/source/general.rst b/source/general.rst new file mode 100644 index 0000000..908a017 --- /dev/null +++ b/source/general.rst @@ -0,0 +1,26 @@ +==================== +General observations +==================== + +License +======= + +The code is licensed under the terms of the GPL version 2 upgrading to GPL 3 +would require consent from all former contributors. Copyright years of files +have not been consistently incremented/updated on changes. + +Languages +========= + +The code base is a mix of Perl, Shell and PHP code. Most of the code is +implemented in PHP. + +Code structure +============== + +Comments and inline documentation +================================= + +The code base is not documented in a good way, there are neither class nor +method or function comments. Comments are just used for the license header +in most of the files. \ No newline at end of file diff --git a/source/glossary.rst b/source/glossary.rst new file mode 100644 index 0000000..b3b2383 --- /dev/null +++ b/source/glossary.rst @@ -0,0 +1,16 @@ +======== +Glossary +======== + +.. glossary:: + + CRL + Definition from :rfc:`5280`: + + X.509 defines one method of certificate revocation. This method + involves each CA periodically issuing a signed data structure called + a certificate revocation list (CRL). A CRL is a time-stamped list + identifying revoked certificates that is signed by a CA or CRL + issuer and made freely available in a public repository. Each + revoked certificate is identified in a CRL by its certificate serial + number. \ No newline at end of file diff --git a/source/index.rst b/source/index.rst index d49e27a..8ecf58f 100644 --- a/source/index.rst +++ b/source/index.rst @@ -20,9 +20,12 @@ contribution. The canonical repository is the :cacertgit:`cacert-devel` though. :maxdepth: 2 :caption: Contents: + general directories database + signer building + glossary Filesystem structure -------------------- @@ -39,5 +42,4 @@ Indices and tables ================== * :ref:`genindex` -* :ref:`modindex` * :ref:`search` diff --git a/source/signer.rst b/source/signer.rst new file mode 100644 index 0000000..299562a --- /dev/null +++ b/source/signer.rst @@ -0,0 +1,222 @@ +=================== +The Signer Protocol +=================== + +Communication with the signer is performed via a serial connection. That has +to be established by the client before speaking the protocol defined here. + +.. _signer-request-data-format: + +Signer request data format specification +======================================== + +Protocol data is encoded with the following format: + +.. table:: signer request message format + + ======= ============================================================== + Byte Data + ======= ============================================================== + 0-2 length of header + data in network byte order + 3-12 action specific header + 13-15 length of first action specific content in network byte order + 15-N fist action specific content string + N+1-N+3 length of second action specific content in network byte order + N+4-M second action specific content string + M+1-M+3 lenght of third action specific content in network byte order + M+4-End third action specific content string + ======= ============================================================== + +Due to the length encoding in 3 bytes the messages can have a maximum length +of 8\ :sup:`3` = 2\ :sup:`24` Bytes which is around 16 MiB. + +General header format +--------------------- + +Every protocol header (bytes 3-12 of protocol message) follows the same 8 byte +structure. The content of bits 3-8 are protocol action specific. + +.. table:: general request header format + + ==== ============================================================= + Byte Value + ==== ============================================================= + 0 Version (``0x01``) + 1 Action + 2 System (``0x01`` for :ref:`client.pl `) + 3 8 bits root + 4 8 bits configuration + 5 8 bits parameter + 6-7 16 bits parameter + 8 8 bits parameter + ==== ============================================================= + +.. _signer-nul-request-format: + +Format of NUL Requests +---------------------- + +NUL requests are sent at the end of each iteration in +:ref:`client.pl `'s main loop. + +.. table:: NUL request header format + + ==== ========================================================= + Byte Value + ==== ========================================================= + 0 Version (``0x01``) + 1 Action ``0x00`` + 2 System (``0x01`` for :ref:`client.pl `) + 3 ``0x00`` + 4 ``0x00`` + 5 ``0x00`` + 6-7 ``0x0000`` + 8 ``0x00`` + ==== ========================================================= + +**NUL Request Payload:** + +- GMT timestamp in %m%d%H%M%Y.%S format +- "" +- "" + +Format of X.509 signing request messages +---------------------------------------- + +X.509 signing request messages are sent in +:ref:`client.pl `'s main loop for each requested +certificate. + +.. table:: X.509 certificate signing request header format + + ==== ========================================================= + Byte Value + ==== ========================================================= + 0 Version (0x01) + 1 Action 0x01 + 2 System (0x01 for :ref:`client.pl `) + 3 Root + 4 Profile (see table :ref:`table-cert-profiles`) + 5 Message Digest Id (see table :ref:`table-md-ids`) + 6-7 Days in big-endian format + 8 Key type (``0x01`` for 'NS', ``0x00`` for others) + ==== ========================================================= + +.. todo:: describe which root is identified by which root id + +The key type is stored in the column *keytype* of the certificate request +table which is one of + +- *domaincerts* +- *emailcerts* +- *orgdomaincerts* +- *orgemailcerts* + +.. todo:: describe what 'NS' means for key type + + +**X.509 Signing Request Payload:** + +- "$content" +- "$SAN" +- "$subject" + +.. _table-cert-profiles: + +.. table:: Certificate profile ids + + == ====================== + Id Profile + == ====================== + 0 Client (personal) + 1 Client (Organization) + 2 Client (Codesigning) + 3 Client (Machine) + 4 Client (ADS) + 5 Server (personal) + 6 Server (Organization) + 7 Server (Jabber) + 8 Server (OCSP) + 9 Server (Timestamp) + 10 Proxy + 11 SubCA + == ====================== + +.. note:: + + :ref:`client.pl ` supports profiles 0, 1, 2, 4, + 5, 6, 8 and 9 only. + +.. _table-md-ids: + +.. table:: Message digest ids + + == ========== + Id Algorithm + == ========== + 1 MD5 + 2 SHA-1 + 3 RIPE-MD160 + 8 SHA-256 + 9 SHA-384 + 10 SHA-512 + == ========== + + +.. todo:: describe other request types + +.. _signer-response-data-format: + +Signer response data format specification +========================================= + +.. todo:: describe signer response + +Protocol messages +================= + +.. _signer-message-handshake: + +Handshake +--------- + +#. client sends 1 byte ``0x02`` to serial port +#. client reads 1 byte from serial port (with a 20 second timeout) +#. client checks whether the byte is ``0x10`` + +.. seqdiag:: + + seqdiag handhake { + client -> server [label = "0x02"]; + client <-- server [label = "0x10"]; + } + +If anything different is received there was a protocol error and no further +messages should be sent over the serial connection. + +Send data +--------- + +:Preconditions: + successful :ref:`Handshake `, + data is encoded according to the :ref:`signer-request-data-format` + +#. client builds byte wise xor of all data bytes into 1 byte $xor +#. client sends concatenated $data string + xor-Byte + "rie4Ech7" +#. client reads 1 byte (with a 5 second timeout) +#. if received byte is ``0x11`` try again +#. if received byte is ``0x10`` the message has been sent successfully + +.. seqdiag:: + + seqdiag with_retry { + client -> client [label = "xor $data"]; + client -> server [label = "$data . $xor . \"rie4Ech7\""]; + client <-- server [label = "0x11"]; + client -> server [label = "$data . $xor . \"rie4Ech7\""]; + client <-- server [label = "0x10"]; + } + +If anything different is received there was a protocol error and no further +messages should be sent over the serial connection. +