Start documenting CommModule

- add a file meant to collect general observations
- add a file meant to collect information related to the database schema
- add a glossary file
- add documentation for the CommModule files in source/directories.rst
- start signer protocol specification in source/signer.rst
- add support for block and sequence diagrams via sphinxcontrib-blockdiag and
  sphinxcontrib-seqdiag
This commit is contained in:
Jan Dittberner 2018-10-28 19:22:32 +01:00
parent e2045ba3c6
commit cff7cc6779
8 changed files with 527 additions and 2 deletions

View file

@ -8,6 +8,9 @@ sphinx = "*"
GitPython = "*"
certifi = "*"
requests = "*"
sphinxcontrib-phpdomain = "*"
sphinxcontrib-blockdiag = "*"
sphinxcontrib-seqdiag = "*"
[dev-packages]

88
Pipfile.lock generated
View file

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "1666cbe0230e5956dfa4b61f4811218f730e7937181a37ab44b32f0270d3bd11"
"sha256": "49436bd593c2b93213655f26a631e356f4630a0358f6515516588831ff8ee25f"
},
"pipfile-spec": 6,
"requires": {
@ -30,6 +30,13 @@
],
"version": "==2.6.0"
},
"blockdiag": {
"hashes": [
"sha256:8dd6570a2ac41b3c0dfe5706de20913cdbebe1bbd2e6dea9ebc13db79df8c151",
"sha256:929125db1cb59145e09dc561021389c7ca71108ef4e4c51a12728eea5b75fccc"
],
"version": "==1.5.4"
},
"certifi": {
"hashes": [
"sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c",
@ -53,6 +60,12 @@
],
"version": "==0.14"
},
"funcparserlib": {
"hashes": [
"sha256:b7992eac1a3eb97b3d91faa342bfda0729e990bd8a43774c1592c091e563c91d"
],
"version": "==0.3.6"
},
"gitdb2": {
"hashes": [
"sha256:83361131a1836661a155172932a13c08bda2db3674e4caa32368aa6eb02f38c2",
@ -102,6 +115,41 @@
],
"version": "==18.0"
},
"pillow": {
"hashes": [
"sha256:00203f406818c3f45d47bb8fe7e67d3feddb8dcbbd45a289a1de7dd789226360",
"sha256:0616f800f348664e694dddb0b0c88d26761dd5e9f34e1ed7b7a7d2da14b40cb7",
"sha256:1f7908aab90c92ad85af9d2fec5fc79456a89b3adcc26314d2cde0e238bd789e",
"sha256:2ea3517cd5779843de8a759c2349a3cd8d3893e03ab47053b66d5ec6f8bc4f93",
"sha256:48a9f0538c91fc136b3a576bee0e7cd174773dc9920b310c21dcb5519722e82c",
"sha256:5280ebc42641a1283b7b1f2c20e5b936692198b9dd9995527c18b794850be1a8",
"sha256:5e34e4b5764af65551647f5cc67cf5198c1d05621781d5173b342e5e55bf023b",
"sha256:63b120421ab85cad909792583f83b6ca3584610c2fe70751e23f606a3c2e87f0",
"sha256:696b5e0109fe368d0057f484e2e91717b49a03f1e310f857f133a4acec9f91dd",
"sha256:870ed021a42b1b02b5fe4a739ea735f671a84128c0a666c705db2cb9abd528eb",
"sha256:916da1c19e4012d06a372127d7140dae894806fad67ef44330e5600d77833581",
"sha256:9303a289fa0811e1c6abd9ddebfc770556d7c3311cb2b32eff72164ddc49bc64",
"sha256:9577888ecc0ad7d06c3746afaba339c94d62b59da16f7a5d1cff9e491f23dace",
"sha256:987e1c94a33c93d9b209315bfda9faa54b8edfce6438a1e93ae866ba20de5956",
"sha256:99a3bbdbb844f4fb5d6dd59fac836a40749781c1fa63c563bc216c27aef63f60",
"sha256:99db8dc3097ceafbcff9cb2bff384b974795edeb11d167d391a02c7bfeeb6e16",
"sha256:a5a96cf49eb580756a44ecf12949e52f211e20bffbf5a95760ac14b1e499cd37",
"sha256:aa6ca3eb56704cdc0d876fc6047ffd5ee960caad52452fbee0f99908a141a0ae",
"sha256:aade5e66795c94e4a2b2624affeea8979648d1b0ae3fcee17e74e2c647fc4a8a",
"sha256:b78905860336c1d292409e3df6ad39cc1f1c7f0964e66844bbc2ebfca434d073",
"sha256:b92f521cdc4e4a3041cc343625b699f20b0b5f976793fb45681aac1efda565f8",
"sha256:bfde84bbd6ae5f782206d454b67b7ee8f7f818c29b99fd02bf022fd33bab14cb",
"sha256:c2b62d3df80e694c0e4a0ed47754c9480521e25642251b3ab1dff050a4e60409",
"sha256:c5e2be6c263b64f6f7656e23e18a4a9980cffc671442795682e8c4e4f815dd9f",
"sha256:c99aa3c63104e0818ec566f8ff3942fb7c7a8f35f9912cb63fd8e12318b214b2",
"sha256:dae06620d3978da346375ebf88b9e2dd7d151335ba668c995aea9ed07af7add4",
"sha256:db5499d0710823fa4fb88206050d46544e8f0e0136a9a5f5570b026584c8fd74",
"sha256:f36baafd82119c4a114b9518202f2a983819101dcc14b26e43fc12cbefdce00e",
"sha256:f52b79c8796d81391ab295b04e520bda6feed54d54931708872e8f9ae9db0ea1",
"sha256:ff8cff01582fa1a7e533cb97f628531c4014af4b5f38e33cdcfe5eec29b6d888"
],
"version": "==5.3.0"
},
"pygments": {
"hashes": [
"sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d",
@ -131,6 +179,13 @@
"index": "pypi",
"version": "==2.20.0"
},
"seqdiag": {
"hashes": [
"sha256:78104e7644c1a4d3a5cacb68de6a7f720793f08dd78561ef0e9e80bed63702bf",
"sha256:887cf56b00bd2492e17ef3a16c4270ff263df3c249eddea85844bb61b594785a"
],
"version": "==0.9.6"
},
"six": {
"hashes": [
"sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
@ -160,6 +215,30 @@
"index": "pypi",
"version": "==1.8.1"
},
"sphinxcontrib-blockdiag": {
"hashes": [
"sha256:2d2ccde16bafb061ae8d2008f9524726e8ccd2a8502651b76a1e7f07a4ffd8eb",
"sha256:7cdff966d8f372b9536374954314a6cf4280e0e48bc2321a4f25cc7f2114f8f0"
],
"index": "pypi",
"version": "==1.5.5"
},
"sphinxcontrib-phpdomain": {
"hashes": [
"sha256:14ecb0b477dddf8ce2c69b72ab15e3455a591d077264808a4f0d35f488b54a5f",
"sha256:ec0286d66233839994a2c812345bbd3f02feca28da941b552bce7d48eb8980f4"
],
"index": "pypi",
"version": "==0.4.1"
},
"sphinxcontrib-seqdiag": {
"hashes": [
"sha256:83c3fdac7e083c5b217f65359c03b75af753209028db6b261b196aff19e7003f",
"sha256:c83f2b552e8e0829dbee22a13c5025f33c0b31a7e87bb589611928c2883d3db5"
],
"index": "pypi",
"version": "==0.8.5"
},
"sphinxcontrib-websupport": {
"hashes": [
"sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd",
@ -173,6 +252,13 @@
"sha256:8819bba37a02d143296a4d032373c4dd4aca11f6d4c9973335ca75f9c8475f59"
],
"version": "==1.24"
},
"webcolors": {
"hashes": [
"sha256:030562f624467a9901f0b455fef05486a88cfb5daa1e356bd4aacea043850b59",
"sha256:b3b88e5ef2b35fa9e01e3fabe99dddf49da074459c44774c59f3ccab3be4f121"
],
"version": "==1.8.1"
}
},
"develop": {}

View file

@ -62,6 +62,9 @@ extensions = [
'sphinx.ext.extlinks',
'sphinx.ext.todo',
'sphinx.ext.ifconfig',
'sphinxcontrib.phpdomain',
'sphinxcontrib.blockdiag',
'sphinxcontrib.seqdiag',
]
# Add any paths that contain templates here, relative to this directory.

View file

@ -14,11 +14,15 @@ The root directory contains
.. _GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0
.. index:: cgi-bin
Directory :file:`cgi-bin`
=========================
The `cgi-bin` directory contains
.. index:: php
.. _cgi-bin-siteseal-cgi:
- :file:`siteseal.cgi` a PHP CGI script that generates some JavaScript code
@ -28,6 +32,169 @@ The `cgi-bin` directory contains
.. todo: check whether this is linked anywhere or can be removed
.. index:: commmodule
.. index:: Perl
.. index:: bash
Directory :file:`CommModule`
============================
This directory contains the CommModule that is implemented in Perl:
.. _commmodule-client-pl:
- :file:`client.pl` the real client, running on the webserver
The style of the Perl code seems a bit inconsistent (mix of uppercase and
lowercase function names, usage of brackets). The code uses database polling
in a loop. It might be a better idea to use some kind of queueing (Redis,
AMQP, ...) to not waste resources when there is nothing to do). Function
parameters are not named which makes the code hard to read.
The script calls several system binaries that need to be present in
compatible versions:
- :program:`openssl`
- :program:`xdelta`
The script uses several Perl standard library modules as well as the
following third party modules:
.. index:: Perl, thirdparty
- `DBD::mysql <https://metacpan.org/pod/DBD::mysql>`_
- `DBI <https://metacpan.org/pod/DBI>`_
- `Device::SerialPort <https://metacpan.org/pod/Device::SerialPort>`_
- `File::CounterFile <https://metacpan.org/pod/File::CounterFile>`_
The script references several openssl configuration files in the HandleCerts
function that are not included in the code repository. There are some
openssl configuration files with similar names in
https://svn.cacert.org/CAcert/SystemAdministration/signer/
The database password is parsed from
:ref:`includes/mysql.php <includes-mysql-php>` and relies on the
exact code that is defined there. Database name, user and host are hardcoded
in the DBI->connect call.
The script implements the client side of the signer protocol which is
specified in :doc:`signer`.
The script performs the following operations:
- parse password from :file:`includes/mysql.php`
- read a list of CRL files and logs their SHA-1 hashes
- read :file:`serial.conf`, create a Device::SerialPort instance `$portObj`,
sets serial parameters and saves :file:`serial.conf`
- run a main loop as long as a file :file:`./client.pl-active` is present.
The main loop performs the following tasks
- handle pending OpenPGP key signing request via ``HandleGPG()``
- handle pending certificate signing requests:
- personal client certificates via ``HandleCerts(0, 0)``
- personal server certificates via ``HandleCerts(0, 1)``
- organization client certificates via ``HandleCerts(1, 0)``
- organization server certificates via ``HandleCerts(1, 1)``
- handle pending certificate revocation requests
- personal client certificates via ``RevokeCerts(0, 0)``
- personal server certificates via ``RevokeCerts(0, 1)``
- organization client certificates via ``RevokeCerts(1, 0)``
- organization server certificates via ``RevokeCerts(1, 1)``
- refresh :term:`CRLs <CRL>` via ``RefreshCRLs()`` in every 100st
iteration
- send a :ref:`NUL request <signer-nul-request-format>` to keep the signer
connection alive
- sleep for 2.7 seconds
There is potential for optimization in the main loop. The CRL update could
be performed if a certificate has been revoked. The NUL request needs only
to be sent if no other request has been sent.
.. todo:: describe more in-depth what each of the main loop steps does
- :file:`commdaemon` a script to run :ref:`client.pl <commmodule-client-pl>`
or :ref:`server.pl <commmodule-server-pl>`
This bash script is automatically restarting the :file:`{script}` given as
the first parameter as long as a file :file:`{script}-active` exists.
Informational messages and errors are logged to syslog via
:command:`logger`.
The script is most probably used to recover from crashed scripts. This
could be implemented via :command:`supervisor` or :command:`systemd`
instead of a custom script.
- :file:`commmodule` a script for startup/shutdown of CommModule from
/etc/init.d
- :file:`logclean.sh` maintenance script for logfiles generated by CommModule
- :file:`serial.conf` serial port configuration file
.. _commmodule-server-pl:
- :file:`server.pl` the real server, running on the signing server
This script implements the signer (server) side of the signer protocol and
performs the actual signing operations.
The script contains a some code that is duplicated by
:ref:`client.pl <commmodule-client-pl>`.
- :file:`usbclient.pl` obsoleted USB version of
:ref:`client.pl <commmodule-client-pl>` above
.. todo: remove unused file (usbclient.pl)
.. todo: add a serial.conf template and move the actual serial.conf into
configuration management
.. todo: clarify why log rotation is implemented with a custom
logclean.sh script instead of using logrotate
Directory :file:`includes`
==============================
.. _includes-mysql-php:
.. _includes-mysql-php-sample:
- :file:`mysql.php.sample` is a template for the database connection handling
code that is meant to be copied to :file:`mysql.php`.
The template defines the MySQL connection as a session variable `mconn` and
tries to connect to that database. It also defines the session variables
`normalhostname`, `securehostname` and `tverify`.
The template defines a function :php:func:`sendmail` for sending mails.
.. php:function:: sendmail($to, $subject, $message, $from, $replyto="", \
$toname="", $fromname="", $errorsto="returns@cacert.org", \
$use_utf8=true)
Send an email. The function reimplements functionality that is readily
available in PHP. The function does not properly escape headers and
sends raw SMTP commands.
:param string $to: recipient email address
:param string $subject: subject
:param string $message: email body
:param string $from: from email address
:param string $replyto: reply-to email address
:param string $fromname: unused in the code
:param string $toname: unused in the code
:param string $errorsto: email address used for Sender and Errors-To
headers
:param bool $use_utf8: decides whether the Content-Type header uses
a charset parameter of utf-8 or iso-8859-1
Configuration and actual code are mixed. It would be better to have a
separate file that just includes configuration.
This file is parsed by :ref:`CommModule/client.pl <commmodule-client-pl>`
format changes might break the CommModule code.
Directory :file:`www`
=====================

26
source/general.rst Normal file
View file

@ -0,0 +1,26 @@
====================
General observations
====================
License
=======
The code is licensed under the terms of the GPL version 2 upgrading to GPL 3
would require consent from all former contributors. Copyright years of files
have not been consistently incremented/updated on changes.
Languages
=========
The code base is a mix of Perl, Shell and PHP code. Most of the code is
implemented in PHP.
Code structure
==============
Comments and inline documentation
=================================
The code base is not documented in a good way, there are neither class nor
method or function comments. Comments are just used for the license header
in most of the files.

16
source/glossary.rst Normal file
View file

@ -0,0 +1,16 @@
========
Glossary
========
.. glossary::
CRL
Definition from :rfc:`5280`:
X.509 defines one method of certificate revocation. This method
involves each CA periodically issuing a signed data structure called
a certificate revocation list (CRL). A CRL is a time-stamped list
identifying revoked certificates that is signed by a CA or CRL
issuer and made freely available in a public repository. Each
revoked certificate is identified in a CRL by its certificate serial
number.

View file

@ -20,9 +20,12 @@ contribution. The canonical repository is the :cacertgit:`cacert-devel` though.
:maxdepth: 2
:caption: Contents:
general
directories
database
signer
building
glossary
Filesystem structure
--------------------
@ -39,5 +42,4 @@ Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

222
source/signer.rst Normal file
View file

@ -0,0 +1,222 @@
===================
The Signer Protocol
===================
Communication with the signer is performed via a serial connection. That has
to be established by the client before speaking the protocol defined here.
.. _signer-request-data-format:
Signer request data format specification
========================================
Protocol data is encoded with the following format:
.. table:: signer request message format
======= ==============================================================
Byte Data
======= ==============================================================
0-2 length of header + data in network byte order
3-12 action specific header
13-15 length of first action specific content in network byte order
15-N fist action specific content string
N+1-N+3 length of second action specific content in network byte order
N+4-M second action specific content string
M+1-M+3 lenght of third action specific content in network byte order
M+4-End third action specific content string
======= ==============================================================
Due to the length encoding in 3 bytes the messages can have a maximum length
of 8\ :sup:`3` = 2\ :sup:`24` Bytes which is around 16 MiB.
General header format
---------------------
Every protocol header (bytes 3-12 of protocol message) follows the same 8 byte
structure. The content of bits 3-8 are protocol action specific.
.. table:: general request header format
==== =============================================================
Byte Value
==== =============================================================
0 Version (``0x01``)
1 Action
2 System (``0x01`` for :ref:`client.pl <commmodule-client-pl>`)
3 8 bits root
4 8 bits configuration
5 8 bits parameter
6-7 16 bits parameter
8 8 bits parameter
==== =============================================================
.. _signer-nul-request-format:
Format of NUL Requests
----------------------
NUL requests are sent at the end of each iteration in
:ref:`client.pl <commmodule-client-pl>`'s main loop.
.. table:: NUL request header format
==== =========================================================
Byte Value
==== =========================================================
0 Version (``0x01``)
1 Action ``0x00``
2 System (``0x01`` for :ref:`client.pl <commmodule-client-pl>`)
3 ``0x00``
4 ``0x00``
5 ``0x00``
6-7 ``0x0000``
8 ``0x00``
==== =========================================================
**NUL Request Payload:**
- GMT timestamp in %m%d%H%M%Y.%S format
- ""
- ""
Format of X.509 signing request messages
----------------------------------------
X.509 signing request messages are sent in
:ref:`client.pl <commmodule-client-pl>`'s main loop for each requested
certificate.
.. table:: X.509 certificate signing request header format
==== =========================================================
Byte Value
==== =========================================================
0 Version (0x01)
1 Action 0x01
2 System (0x01 for :ref:`client.pl <commmodule-client-pl>`)
3 Root
4 Profile (see table :ref:`table-cert-profiles`)
5 Message Digest Id (see table :ref:`table-md-ids`)
6-7 Days in big-endian format
8 Key type (``0x01`` for 'NS', ``0x00`` for others)
==== =========================================================
.. todo:: describe which root is identified by which root id
The key type is stored in the column *keytype* of the certificate request
table which is one of
- *domaincerts*
- *emailcerts*
- *orgdomaincerts*
- *orgemailcerts*
.. todo:: describe what 'NS' means for key type
**X.509 Signing Request Payload:**
- "$content"
- "$SAN"
- "$subject"
.. _table-cert-profiles:
.. table:: Certificate profile ids
== ======================
Id Profile
== ======================
0 Client (personal)
1 Client (Organization)
2 Client (Codesigning)
3 Client (Machine)
4 Client (ADS)
5 Server (personal)
6 Server (Organization)
7 Server (Jabber)
8 Server (OCSP)
9 Server (Timestamp)
10 Proxy
11 SubCA
== ======================
.. note::
:ref:`client.pl <commmodule-client-pl>` supports profiles 0, 1, 2, 4,
5, 6, 8 and 9 only.
.. _table-md-ids:
.. table:: Message digest ids
== ==========
Id Algorithm
== ==========
1 MD5
2 SHA-1
3 RIPE-MD160
8 SHA-256
9 SHA-384
10 SHA-512
== ==========
.. todo:: describe other request types
.. _signer-response-data-format:
Signer response data format specification
=========================================
.. todo:: describe signer response
Protocol messages
=================
.. _signer-message-handshake:
Handshake
---------
#. client sends 1 byte ``0x02`` to serial port
#. client reads 1 byte from serial port (with a 20 second timeout)
#. client checks whether the byte is ``0x10``
.. seqdiag::
seqdiag handhake {
client -> server [label = "0x02"];
client <-- server [label = "0x10"];
}
If anything different is received there was a protocol error and no further
messages should be sent over the serial connection.
Send data
---------
:Preconditions:
successful :ref:`Handshake <signer-message-handshake>`,
data is encoded according to the :ref:`signer-request-data-format`
#. client builds byte wise xor of all data bytes into 1 byte $xor
#. client sends concatenated $data string + xor-Byte + "rie4Ech7"
#. client reads 1 byte (with a 5 second timeout)
#. if received byte is ``0x11`` try again
#. if received byte is ``0x10`` the message has been sent successfully
.. seqdiag::
seqdiag with_retry {
client -> client [label = "xor $data"];
client -> server [label = "$data . $xor . \"rie4Ech7\""];
client <-- server [label = "0x11"];
client -> server [label = "$data . $xor . \"rie4Ech7\""];
client <-- server [label = "0x10"];
}
If anything different is received there was a protocol error and no further
messages should be sent over the serial connection.