jandd/django-cats
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Cleanup data model

Browse source
This commit is contained in:
Jan Dittberner 2024-09-20 10:57:25 +02:00
parent f41126894a
commit 0b7d001062
12 changed files with 646 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
cats/migrations/0016_improve_user_field_naming.py Normal file
View file

@ -0,0 +1,32 @@
# Generated by Django 4.2.16 on 2024-09-19 08:55
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("cats", "0015_refactor_answer_model"),
]
operations = [
migrations.RenameField(
model_name="user",
old_name="cn_name",
new_name="common_name",
),
migrations.RenameField(
model_name="user",
old_name="root",
new_name="issuer_name",
),
migrations.RenameField(
model_name="user",
old_name="user_id",
new_name="serial_number",
),
migrations.AlterUniqueTogether(
name="user",
unique_together={("serial_number", "issuer_name")},
),
]

16
cats/migrations/0017_delete_useraddress.py Normal file
View file

@ -0,0 +1,16 @@
# Generated by Django 4.2.16 on 2024-09-19 08:58
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("cats", "0016_improve_user_field_naming"),
]
operations = [
migrations.DeleteModel(
name="UserAddress",
),
]

44
cats/migrations/0018_refactor_user_handling.py Normal file
View file

@ -0,0 +1,44 @@
# Generated by Django 4.2.16 on 2024-09-20 06:43
from django.conf import settings
import django.contrib.auth.models
import django.contrib.auth.validators
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
class Migration(migrations.Migration):
dependencies = [
("cats", "0017_delete_useraddress"),
]
operations = [
migrations.RenameField(
model_name="learnprogress",
old_name="root",
new_name="issuer_name",
),
migrations.RenameField(
model_name="learnprogress",
old_name="user_id",
new_name="serial_number",
),
migrations.AlterField(
model_name="learnprogress",
name="t_id",
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="cats.topic"
),
),
migrations.AddField(
model_name="learnprogress",
name="user",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
to=settings.AUTH_USER_MODEL,
),
),
]

108
cats/migrations/0019_prepare_to_separate_certificate_information.py Normal file
View file

@ -0,0 +1,108 @@
# Generated by Django 4.2.16 on 2024-09-20 07:21
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
("cats", "0018_refactor_user_handling"),
]
operations = [
migrations.CreateModel(
name="UserProfile",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"language",
models.CharField(
help_text="preferred language of the user ISO 639 2 letter code",
max_length=2,
),
),
(
"send_certificate",
models.CharField(
choices=[
("no", "no"),
("email", "via email"),
("post", "via postal mail"),
],
max_length=13,
),
),
(
"user",
models.OneToOneField(
on_delete=django.db.models.deletion.CASCADE,
to=settings.AUTH_USER_MODEL,
),
),
],
),
migrations.CreateModel(
name="UserCertificate",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"serial_number",
models.CharField(
help_text="RFC-5280 4.1.2.2. Serial Number hexadecimal without prefix",
max_length=20,
),
),
(
"issuer_name",
models.CharField(
help_text="Common Name of the Issuer DN field", max_length=100
),
),
(
"common_name",
models.CharField(
help_text="Common Name of the Subject DN field", max_length=100
),
),
(
"user",
models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
to=settings.AUTH_USER_MODEL,
),
),
],
options={
"unique_together": {("serial_number", "issuer_name")},
},
),
migrations.AddField(
model_name="learnprogress",
name="user_certificate",
field=models.ForeignKey(
null=True,
on_delete=django.db.models.deletion.CASCADE,
to="cats.usercertificate",
),
),
]

72
cats/migrations/0020_user_certificate_data.py Normal file
View file

@ -0,0 +1,72 @@
# Generated by Django 4.2.16 on 2024-09-20 07:23
import logging
from django.db import migrations
def fill_user_certificate_data(apps, schema_editor):
OriginalUser = apps.get_model("cats", "User")
UserProfile = apps.get_model("cats", "UserProfile")
UserCertificate = apps.get_model("cats", "UserCertificate")
AuthUser = apps.get_model("auth", "User")
for user in OriginalUser.objects.all():
auth_user_id = None
if (
user.email
and "@" in user.email
and not AuthUser.objects.filter(email=user.email).exists()
):
auth_user = AuthUser.objects.create(
username=user.email,
email=user.email,
is_staff=user.admin == "1",
)
UserProfile.objects.create(
user=auth_user,
language=user.lang,
send_certificate=user.send_certificate,
)
auth_user_id = auth_user.id
UserCertificate.objects.create(
user_id=auth_user_id,
serial_number=user.serial_number,
issuer_name=user.issuer_name,
common_name=user.common_name,
)
def fill_learn_progress_references(apps, schema_editor):
UserCertificate = apps.get_model("cats", "UserCertificate")
LearnProgress = apps.get_model("cats", "LearnProgress")
logger = logging.getLogger(__name__)
for learn_progress in LearnProgress.objects.all():
try:
user_cert = UserCertificate.objects.get(
serial_number=learn_progress.serial_number,
issuer_name=learn_progress.issuer_name,
)
learn_progress.user_certificate = user_cert
if user_cert.user_id:
learn_progress.user_id = user_cert.user_id
learn_progress.save()
except UserCertificate.DoesNotExist:
logger.warning(
"no user certificate found for learn_progress %d", learn_progress.id
)
class Migration(migrations.Migration):
dependencies = [
("cats", "0019_prepare_to_separate_certificate_information"),
]
operations = [
# remove invalid learn progress entities
migrations.RunSQL("DELETE FROM cats_learnprogress WHERE t_id_id NOT IN (SELECT id FROM cats_topic)"),
migrations.RunPython(fill_user_certificate_data),
migrations.RunPython(fill_learn_progress_references),
]

50
cats/migrations/0021_remove_redundancy.py Normal file
View file

@ -0,0 +1,50 @@
# Generated by Django 4.2.16 on 2024-09-20 08:09
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
("cats", "0020_user_certificate_data"),
]
operations = [
migrations.DeleteModel(
name="User",
),
migrations.RenameField(
model_name="learnprogress",
old_name="t_id",
new_name="topic",
),
migrations.RemoveField(
model_name="learnprogress",
name="issuer_name",
),
migrations.RemoveField(
model_name="learnprogress",
name="serial_number",
),
migrations.AlterField(
model_name="learnprogress",
name="passed",
field=models.SmallIntegerField(),
),
migrations.AlterField(
model_name="learnprogress",
name="uploaded",
field=models.BooleanField(
default=False,
help_text="indicates whether the learn progress has been uploaded to the main CAcert web application",
),
),
migrations.AlterField(
model_name="learnprogress",
name="user_certificate",
field=models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="cats.usercertificate"
),
),
]

17
cats/migrations/0022_rename_incorrect_answers.py Normal file
View file

@ -0,0 +1,17 @@
# Generated by Django 4.2.16 on 2024-09-20 08:15
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("cats", "0021_remove_redundancy"),
]
operations = [
migrations.RenameModel(
old_name="IncorrectAnswer",
new_name="OriginalIncorrectAnswer",
),
]

56
cats/migrations/0023_fill_new_incorrect_answers.py Normal file
View file

@ -0,0 +1,56 @@
# Generated by Django 4.2.16 on 2024-09-20 08:20
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
("cats", "0022_rename_incorrect_answers"),
]
operations = [
migrations.CreateModel(
name="IncorrectAnswer",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"learn_progress",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="cats.learnprogress",
),
),
(
"question",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="cats.question"
),
),
],
options={
"unique_together": {("learn_progress", "question")},
},
),
# copy data from old entity without proper primary key, skipping invalid data that would violate
# the foreign key constraints on the new table
migrations.RunSQL(
"""
INSERT INTO cats_incorrectanswer (learn_progress_id, question_id)
SELECT distinct ia.lp_id, ia.q_id
FROM cats_originalincorrectanswer ia
JOIN cats_learnprogress lp on lp.id=ia.lp_id
JOIN cats_question q ON q.id=ia.q_id;
"""
),
migrations.DeleteModel("OriginalIncorrectAnswer"),
]

35
cats/migrations/0024_refactor_statistics_table.py Normal file
View file

@ -0,0 +1,35 @@
# Generated by Django 4.2.16 on 2024-09-20 08:29
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
("cats", "0023_fill_new_incorrect_answers"),
]
operations = [
# remove invalid entries
migrations.RunSQL("DELETE FROM cats_statistics WHERE q_id NOT IN (SELECT id FROM cats_question)"),
migrations.AlterField(
model_name="statistics",
name="q_id",
field=models.ForeignKey(
help_text="question",
on_delete=django.db.models.deletion.CASCADE,
to="cats.question",
),
),
migrations.RenameField(
model_name="statistics",
old_name="q_id",
new_name="question",
),
migrations.AlterField(
model_name="statistics",
name="count",
field=models.IntegerField(help_text="count of answers"),
),
]

119
cats/migrations/0025_cleanup_after_schema_normalization.py Normal file
View file

@ -0,0 +1,119 @@
# Generated by Django 4.2.16 on 2024-09-20 08:43
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
("cats", "0024_refactor_statistics_table"),
]
operations = [
migrations.DeleteModel(
name="SchemaVersion",
),
migrations.DeleteModel(
name="Temp",
),
migrations.AlterModelOptions(
name="question",
options={"verbose_name": "question"},
),
migrations.AlterModelOptions(
name="questiontype",
options={"verbose_name": "question type"},
),
migrations.AlterModelOptions(
name="questiontypename",
options={"verbose_name": "question type name"},
),
migrations.AlterModelOptions(
name="topic",
options={"verbose_name": "topic"},
),
migrations.AlterModelOptions(
name="topictype",
options={"verbose_name": "topic type"},
),
migrations.AlterField(
model_name="answer",
name="reference_answer",
field=models.ForeignKey(
blank=True,
help_text="referenced answer in original topic",
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="referenced",
to="cats.answer",
),
),
migrations.AlterField(
model_name="learnprogress",
name="correct",
field=models.IntegerField(help_text="questions with correct answer"),
),
migrations.AlterField(
model_name="learnprogress",
name="date",
field=models.DateTimeField(help_text="time and date"),
),
migrations.AlterField(
model_name="learnprogress",
name="number",
field=models.IntegerField(help_text="number of questions"),
),
migrations.AlterField(
model_name="learnprogress",
name="passed",
field=models.SmallIntegerField(
choices=[(-1, "not finished"), (0, "not passed"), (1, "passed")],
default=-1,
help_text="-1 means not finished, 0 means not passed, 1 means passed",
),
),
migrations.AlterField(
model_name="learnprogress",
name="wrong",
field=models.IntegerField(help_text="questions with wrong answer"),
),
migrations.AlterField(
model_name="learnprogress",
name="percentage",
field=models.DecimalField(
blank=True,
decimal_places=0,
help_text="percentage of questions that have been answered correctly",
max_digits=5,
null=True,
),
),
migrations.AlterField(
model_name="question",
name="question",
field=models.TextField(help_text="question"),
),
migrations.AlterField(
model_name="question",
name="reference_question",
field=models.ForeignKey(
blank=True,
help_text="referenced question in original topic",
null=True,
on_delete=django.db.models.deletion.CASCADE,
related_name="referenced",
to="cats.question",
),
),
migrations.AlterField(
model_name="topic",
name="number_of_questions",
field=models.IntegerField(),
),
migrations.AlterField(
model_name="topic",
name="topic",
field=models.CharField(max_length=50, unique=True),
),
]

173
cats/models.py
View file

@ -1,34 +1,44 @@
# This is an auto-generated Django model module.
# You'll have to do the following manually to clean this up:
# * Rearrange models' order
# * Make sure each model has one field with primary_key=True
# * Make sure each ForeignKey and OneToOneField has `on_delete` set to the desired behavior
# * Remove `managed = False` lines if you wish to allow Django to create, modify, and delete the table
# Feel free to rename the models, but don't rename db_table values or field names.
from django.conf import settings
from django.conf.global_settings import AUTH_USER_MODEL
from django.contrib.auth.models import AbstractUser
from django.db import models
from django.utils.translation import gettext_lazy as _
class TopicType(models.Model):
text = models.CharField(max_length=255)
class Meta:
verbose_name = _("topic type")
def __str__(self):
return self.text
class Topic(models.Model):
topic = models.CharField(unique=True, max_length=50, db_comment="Thema")
topic = models.CharField(unique=True, max_length=50)
active = models.IntegerField()
number_of_questions = models.IntegerField(
db_column="numOfQu"
) # Field name made lowercase.
number_of_questions = models.IntegerField()
percentage = models.IntegerField()
lang = models.CharField(max_length=42, blank=True, null=True)
topic_type = models.ForeignKey(TopicType, on_delete=models.CASCADE)
class Meta:
verbose_name = "Topic"
verbose_name = _("topic")
def __str__(self):
return f"{self.topic} ({self.topic_type}, {self.lang})"
class QuestionType(models.Model):
class Meta:
verbose_name = "Question Type"
verbose_name = _("question type")
def get_name(self, language: str) -> str:
if self.questiontypename_set.filter(lang=language).exists():
return self.questiontypename_set.get(lang=language).name
return self.questiontypename_set.get(lang="en").name
class QuestionTypeName(models.Model):
@ -37,14 +47,17 @@ class QuestionTypeName(models.Model):
name = models.CharField(max_length=25)
class Meta:
verbose_name = "Question Type Name"
verbose_name = _("question type name")
unique_together = (("question_type", "lang"),)
def __str__(self):
return f"{self.name} ({self.lang})"
class Question(models.Model):
question_type = models.ForeignKey(QuestionType, on_delete=models.CASCADE)
topic = models.ForeignKey(Topic, on_delete=models.CASCADE)
question = models.TextField(db_comment="Frage")
question = models.TextField(help_text=_("question"))
active = models.CharField(max_length=1)
description = models.CharField(max_length=1)
reference_question = models.ForeignKey(
@ -53,12 +66,16 @@ class Question(models.Model):
on_delete=models.CASCADE,
null=True,
blank=True,
help_text=_("referenced question in original topic")
)
translation_status = models.IntegerField(blank=True, null=True)
explanation = models.TextField(blank=True)
class Meta:
verbose_name = "Question"
verbose_name = _("question")
def __str__(self):
return f'"{self.question}" ({self.question_type.get_name(self.topic.lang)})'
class Answer(models.Model):
@ -71,81 +88,81 @@ class Answer(models.Model):
on_delete=models.CASCADE,
null=True,
blank=True,
help_text=_("referenced answer in original topic"),
)
def __str__(self):
return f"\"{self.answer}\" ({'correct' if self.correct else 'incorrect'})"
class User(models.Model):
user_id = models.CharField(
primary_key=True, max_length=10
) # The composite primary key (user_id, root) found, that is not supported. The first column is selected.
cn_name = models.CharField(
db_column="CN_name", max_length=100
) # Field name made lowercase.
lang = models.CharField(max_length=2)
admin = models.CharField(max_length=1)
email = models.CharField(max_length=100)
class UserCertificate(models.Model):
user = models.ForeignKey(AUTH_USER_MODEL, on_delete=models.CASCADE, null=True)
serial_number = models.CharField(
max_length=20,
help_text=_("RFC-5280 4.1.2.2. Serial Number hexadecimal without prefix"),
)
issuer_name = models.CharField(
max_length=100, help_text=_("Common Name of the Issuer DN field")
)
common_name = models.CharField(
max_length=100, help_text=_("Common Name of the Subject DN field")
)
class Meta:
unique_together = (("serial_number", "issuer_name"),)
class UserProfile(models.Model):
user = models.OneToOneField(AUTH_USER_MODEL, on_delete=models.CASCADE)
language = models.CharField(
max_length=2,
help_text=_("preferred language of the user ISO 639 2 letter code"),
)
send_certificate = models.CharField(
db_column="sendCert", max_length=13
) # Field name made lowercase.
root = models.CharField(max_length=45)
class Meta:
unique_together = (("user_id", "root"),)
class UserAddress(models.Model):
user_id = models.CharField(
primary_key=True, max_length=10
) # The composite primary key (user_id, root) found, that is not supported. The first column is selected.
root = models.CharField(max_length=45)
firstname = models.CharField(max_length=25)
lastname = models.CharField(max_length=25)
street = models.CharField(max_length=50)
house_number = models.CharField(max_length=5)
zipcode = models.CharField(max_length=10)
city = models.CharField(max_length=30)
state = models.CharField(max_length=50)
country = models.CharField(max_length=50)
class Meta:
unique_together = (("user_id", "root"),)
max_length=13,
choices=(
("no", _("no")),
("email", _("via email")),
("post", _("via postal mail")),
),
)
class LearnProgress(models.Model):
user_id = models.CharField(max_length=15)
root = models.CharField(max_length=45)
date = models.DateTimeField(db_comment="Uhrzeit und Datum")
t_id = models.IntegerField(db_comment="Themen ID")
number = models.IntegerField(db_comment="Anzahl der Fragen")
correct = models.IntegerField(db_comment="Richtige Fragen")
wrong = models.IntegerField(db_comment="Anzahl der falschen Antworten")
percentage = models.DecimalField(
max_digits=5, decimal_places=0, blank=True, null=True
user = models.ForeignKey(
settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True
)
user_certificate = models.ForeignKey(
UserCertificate, on_delete=models.CASCADE,
)
date = models.DateTimeField(help_text=_("time and date"))
topic = models.ForeignKey(Topic, on_delete=models.CASCADE)
number = models.IntegerField(help_text=_("number of questions"))
correct = models.IntegerField(help_text=_("questions with correct answer"))
wrong = models.IntegerField(help_text=_("questions with wrong answer"))
percentage = models.DecimalField(
max_digits=5, decimal_places=0, blank=True, null=True,
help_text=_("percentage of questions that have been answered correctly"),
)
uploaded = models.BooleanField(
help_text=_("indicates whether the learn progress has been uploaded to the main CAcert web application"),
default=False,
)
passed = models.SmallIntegerField(
help_text=_("-1 means not finished, 0 means not passed, 1 means passed"),
choices=((-1, _("not finished")), (0, _("not passed")), (1, _("passed"))),
default=-1,
)
uploaded = models.IntegerField(blank=True, null=True)
passed = models.IntegerField()
class IncorrectAnswer(models.Model):
lp_id = models.IntegerField(
primary_key=True
) # The composite primary key (lp_id, q_id) found, that is not supported. The first column is selected.
q_id = models.IntegerField()
learn_progress = models.ForeignKey(LearnProgress, on_delete=models.CASCADE)
question = models.ForeignKey(Question, on_delete=models.CASCADE)
class Meta:
unique_together = (("lp_id", "q_id"),)
class SchemaVersion(models.Model):
version = models.IntegerField(unique=True)
when = models.DateTimeField()
unique_together = (("learn_progress", "question"),)
class Statistics(models.Model):
q_id = models.IntegerField(db_comment="Frage Id")
count = models.IntegerField(db_comment="Zählen von Antworten")
class Temp(models.Model):
uid = models.CharField(max_length=10, blank=True, null=True)
number = models.IntegerField(blank=True, null=True)
question = models.ForeignKey(Question, on_delete=models.CASCADE, help_text=_("question"))
count = models.IntegerField(help_text=_("count of answers"))

2
django_cats/settings.py
View file

@ -60,6 +60,8 @@ MIDDLEWARE = [
ROOT_URLCONF = "django_cats.urls"
CATS_ADMIN_EMAILS = env.list("CATS_ADMIN_EMAILS", default=[])
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",