Provide a CSV with a pid column, whose value in each row must match + the PID of an existing volume. Additional columns will be used to update the volume's + metadata.
+Columns matching Manifest model field names will update those + fields directly, and any additional columns will be used to populate the volume's + metadata JSON field.
+ """, + ) + + def clean(self): + # check csv has pid column + super().clean() + csv_file = self.cleaned_data.get("csv_file") + if csv_file: + reader = csv.DictReader( + normalize_header(StringIO(csv_file.read().decode("utf-8"))), + ) + if "pid" not in reader.fieldnames: + self.add_error( + "metadata_spreadsheet", + forms.ValidationError( + """Spreadsheet must have pid column. Check to ensure there + are no stray characters in the header row.""" + ), + ) + # return back to start of file so we can read again + csv_file.seek(0, 0) + return self.cleaned_data diff --git a/apps/iiif/manifests/migrations/0053_auto_20220607_1453.py b/apps/iiif/manifests/migrations/0053_auto_20220607_1453.py new file mode 100644 index 000000000..3ee44bddc --- /dev/null +++ b/apps/iiif/manifests/migrations/0053_auto_20220607_1453.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.13 on 2022-06-07 14:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('manifests', '0052_alter_imageserver_storage_path'), + ] + + operations = [ + migrations.AddField( + model_name='manifest', + name='modified_at', + field=models.DateTimeField(auto_now=True, null=True), + ), + migrations.AlterField( + model_name='manifest', + name='label', + field=models.CharField(default='', max_length=1000), + ), + ] diff --git a/apps/iiif/manifests/migrations/0054_manifest_logo_url.py b/apps/iiif/manifests/migrations/0054_manifest_logo_url.py new file mode 100644 index 000000000..684570f42 --- /dev/null +++ b/apps/iiif/manifests/migrations/0054_manifest_logo_url.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.17 on 2023-02-16 20:43 + +from django.db import migrations, models +from django.conf import settings + +class Migration(migrations.Migration): + + dependencies = [ + ('manifests', '0053_auto_20220607_1453'), + ] + + operations = [ + migrations.AddField( + model_name='manifest', + name='logo_url', + field=models.URLField(blank=True, help_text="URL for thumbnail of holding institution's logo"), + ), + ] diff --git a/apps/iiif/manifests/migrations/0055_alter_manifest_logo_url.py b/apps/iiif/manifests/migrations/0055_alter_manifest_logo_url.py new file mode 100644 index 000000000..10f717df5 --- /dev/null +++ b/apps/iiif/manifests/migrations/0055_alter_manifest_logo_url.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.12 on 2023-04-05 21:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('manifests', '0054_manifest_logo_url'), + ] + + operations = [ + migrations.AlterField( + model_name='manifest', + name='logo_url', + field=models.URLField(blank=True, help_text='A URL in this field will take precedence over any logo file uploaded in the file field. Clear this field (if populated) to use an uploaded logo file instead.'), + ), + ] diff --git a/apps/iiif/manifests/migrations/0056_alter_manifest_metadata.py b/apps/iiif/manifests/migrations/0056_alter_manifest_metadata.py new file mode 100644 index 000000000..3a4d104dd --- /dev/null +++ b/apps/iiif/manifests/migrations/0056_alter_manifest_metadata.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2023-09-25 20:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('manifests', '0055_alter_manifest_logo_url'), + ] + + operations = [ + migrations.AlterField( + model_name='manifest', + name='metadata', + field=models.JSONField(blank=True, default=dict), + ), + ] diff --git a/apps/iiif/manifests/migrations/0057_alter_manifest_languages.py b/apps/iiif/manifests/migrations/0057_alter_manifest_languages.py new file mode 100644 index 000000000..275faba85 --- /dev/null +++ b/apps/iiif/manifests/migrations/0057_alter_manifest_languages.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2023-09-25 21:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('manifests', '0056_alter_manifest_metadata'), + ] + + operations = [ + migrations.AlterField( + model_name='manifest', + name='languages', + field=models.ManyToManyField(blank=True, help_text='Languages present in the manifest.', to='manifests.Language'), + ), + ] diff --git a/apps/iiif/manifests/migrations/0058_alter_relatedlink.py b/apps/iiif/manifests/migrations/0058_alter_relatedlink.py new file mode 100644 index 000000000..cdb82b69d --- /dev/null +++ b/apps/iiif/manifests/migrations/0058_alter_relatedlink.py @@ -0,0 +1,59 @@ +# Generated by Django 3.2.12 on 2023-12-05 16:16 + +from django.db import migrations, models +import uuid + + +def populate_is_structured_data(apps, schema_editor): + # Data migration to populate RelatedLink.is_structured_data for existing data + RelatedLink = apps.get_model("manifests", "RelatedLink") + rl_set = RelatedLink.objects.all() + for rl in rl_set: + # Assume all existing RelatedLinks are structured data, since they were previously only + # being added in Remote ingests when there was an existing manifest (which is structured + # data) + rl.is_structured_data = True + RelatedLink.objects.bulk_update(rl_set, ["is_structured_data"], batch_size=1000) + + +class Migration(migrations.Migration): + dependencies = [ + ("manifests", "0057_alter_manifest_languages"), + ] + + operations = [ + migrations.AlterField( + model_name="relatedlink", + name="id", + field=models.UUIDField( + default=uuid.uuid4, editable=False, primary_key=True, serialize=False + ), + ), + migrations.AlterField( + model_name="relatedlink", + name="format", + field=models.CharField( + blank=True, + choices=[ + ("text/html", "HTML or web page"), + ("application/json", "JSON"), + ("application/ld+json", "JSON-LD"), + ("application/pdf", "PDF"), + ("text/plain", "Text"), + ("application/xml", "XML"), + ("application/octet-stream", "Other"), + ], + max_length=255, + null=True, + ), + ), + migrations.AddField( + model_name="relatedlink", + name="is_structured_data", + field=models.BooleanField( + default=False, + help_text="True if this link is structured data that should appear in the manifest's 'seeAlso' field; if false, the link will appear in the 'related' field instead. Leave unchecked if unsure.", + ), + ), + migrations.RunPython(populate_is_structured_data, migrations.RunPython.noop), + ] diff --git a/apps/iiif/manifests/migrations/0059_auto_20250113_1401.py b/apps/iiif/manifests/migrations/0059_auto_20250113_1401.py new file mode 100644 index 000000000..0749f6cce --- /dev/null +++ b/apps/iiif/manifests/migrations/0059_auto_20250113_1401.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.23 on 2025-01-13 14:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('manifests', '0058_alter_relatedlink'), + ] + + operations = [ + migrations.AddField( + model_name='manifest', + name='searchable', + field=models.BooleanField(default=True), + ), + migrations.AlterField( + model_name='imageserver', + name='storage_service', + field=models.CharField(choices=[('sftp', 'SFTP'), ('s3', 'S3'), ('remote', 'Remote'), ('local', 'Local')], default='sftp', max_length=10), + ), + ] diff --git a/apps/iiif/manifests/models.py b/apps/iiif/manifests/models.py index 2b817558b..378ddaca6 100644 --- a/apps/iiif/manifests/models.py +++ b/apps/iiif/manifests/models.py @@ -1,10 +1,12 @@ """Django models for IIIF manifests""" -from boto3 import resource + +from os import environ from uuid import uuid4, UUID from json import JSONEncoder +from boto3 import resource +from urllib.parse import urlparse from django.apps import apps from django.db import models -from django.contrib.postgres.fields import JSONField from django.contrib.postgres.search import SearchVectorField, SearchVector from django.contrib.postgres.aggregates import StringAgg from django.contrib.contenttypes.fields import GenericForeignKey, GenericRelation @@ -12,42 +14,49 @@ from django.conf import settings from edtf.fields import EDTFField from apps.iiif.manifests.validators import validate_edtf -import config.settings.local as settings from ..choices import Choices from ..kollections.models import Collection -from..models import IiifBase -JSONEncoder_olddefault = JSONEncoder.default # pylint: disable = invalid-name +from ..models import IiifBase +from .tasks import index_manifest_task + +JSONEncoder_olddefault = JSONEncoder.default # pylint: disable = invalid-name + -def JSONEncoder_newdefault(self, o): # pylint: disable = invalid-name +def JSONEncoder_newdefault(self, o): # pylint: disable = invalid-name """This JSONEncoder makes Wagtail autocomplete run - do not delete.""" if isinstance(o, UUID): return str(o) return JSONEncoder_olddefault(self, o) + + JSONEncoder.default = JSONEncoder_newdefault + class ImageServer(models.Model): """Django model for IIIF image server info. Each canvas has one ImageServer""" STORAGE_SERVICES = ( - ('sftp', 'SFTP'), - ('s3', 'S3'), - ('remote', 'Remote'), + ("sftp", "SFTP"), + ("s3", "S3"), + ("remote", "Remote"), + ("local", "Local"), ) id = models.UUIDField(primary_key=True, default=uuid4) server_base = models.CharField( - max_length=255, - default=settings.IIIF_IMAGE_SERVER_BASE + max_length=255, default=settings.IIIF_IMAGE_SERVER_BASE + ) + storage_service = models.CharField( + max_length=10, choices=STORAGE_SERVICES, default="sftp" ) - storage_service = models.CharField(max_length=10, choices=STORAGE_SERVICES, default='sftp') - storage_path = models.CharField(max_length=255, default='') + storage_path = models.CharField(max_length=255, default="") sftp_user = models.CharField(max_length=100, null=True, blank=True) sftp_port = models.IntegerField(default=22) - private_key_path = models.CharField(max_length=500, default='~/.ssh/id_rsa.pem') - path_delineator = models.CharField(max_length=10, default='/') + private_key_path = models.CharField(max_length=500, default="~/.ssh/id_rsa.pem") + path_delineator = models.CharField(max_length=10, default="/") def __str__(self): - return f'{self.server_base}' + return f"{self.server_base}" @property def bucket(self): @@ -57,8 +66,8 @@ def bucket(self): :return: S3 bucket :rtype: boto3.resources.factory.s3.Bucket """ - if self.storage_service == 's3': - s3 = resource('s3') + if self.storage_service == "s3": + s3 = resource("s3") return s3.Bucket(self.storage_path) return None @@ -71,24 +80,30 @@ def sftp_connection(self): :rtype: dict """ return { - 'host': self.server_base, - 'username': self.sftp_user, - 'private_key': self.private_key_path, - 'port': self.sftp_port, - 'default_path': self.storage_path + "host": urlparse(self.server_base).netloc, + "username": self.sftp_user, + "private_key": self.private_key_path, + "port": self.sftp_port, + "default_path": self.storage_path, } + class ValueByLanguage(models.Model): - """ Labels by language. """ + """Labels by language.""" + id = models.UUIDField(primary_key=True, default=uuid4) - language = models.CharField(max_length=16, choices=Choices.LANGUAGES, default=settings.DEFAULT_LANGUAGE) + language = models.CharField( + max_length=16, choices=Choices.LANGUAGES, default=settings.DEFAULT_LANGUAGE + ) content = models.TextField() content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) object_id = models.UUIDField() - content_object = GenericForeignKey('content_type', 'object_id') + content_object = GenericForeignKey("content_type", "object_id") + class Language(models.Model): """Model to store language names and codes for multiple choice fields""" + code = models.CharField(max_length=16, unique=True) name = models.CharField(max_length=255) @@ -97,10 +112,12 @@ class Meta: def __str__(self): """String representation of the language""" - return self.name + return str(self.name) -class ManifestManager(models.Manager): # pylint: disable = too-few-public-methods + +class ManifestManager(models.Manager): # pylint: disable = too-few-public-methods """Model manager for searches.""" + def with_documents(self): """[summary] @@ -109,29 +126,34 @@ def with_documents(self): :return: [description] :rtype: django.db.models.QuerySet """ - vector = SearchVector(StringAgg('canvas__annotation__content', delimiter=' ')) + vector = SearchVector(StringAgg("canvas__annotation__content", delimiter=" ")) return self.get_queryset().annotate(document=vector) + class Manifest(IiifBase): """Model class for IIIF Manifest""" DIRECTIONS = ( - ('left-to-right', 'Left to Right'), - ('right-to-left', 'Right to Left') + ("left-to-right", "Left to Right"), + ("right-to-left", "Right to Left"), ) summary = models.TextField(null=True, blank=True) - author = models.TextField(null=True, blank=True, help_text="Enter multiple entities separated by a semicolon (;).") + author = models.TextField( + null=True, + blank=True, + help_text="Enter multiple entities separated by a semicolon (;).", + ) published_city = models.TextField( null=True, blank=True, - help_text="Enter multiple entities separated by a semicolon (;)." + help_text="Enter multiple entities separated by a semicolon (;).", ) published_date = models.CharField( "Published date (display)", max_length=255, null=True, blank=True, - help_text="Used for display only." + help_text="Used for display only.", ) published_date_edtf = models.CharField( # Character field editable in admin "Published date (EDTF)", @@ -145,11 +167,11 @@ class Manifest(IiifBase): ) date_edtf = EDTFField( # Read-only EDTF field that handles fuzzy date calculations "Date of publication (EDTF)", - natural_text_field='published_date_edtf', - lower_fuzzy_field='date_earliest', - upper_fuzzy_field='date_latest', - lower_strict_field='date_sort_ascending', - upper_strict_field='date_sort_descending', + natural_text_field="published_date_edtf", + lower_fuzzy_field="date_earliest", + upper_fuzzy_field="date_latest", + lower_strict_field="date_sort_ascending", + upper_strict_field="date_sort_descending", blank=True, null=True, ) @@ -159,49 +181,76 @@ class Manifest(IiifBase): # use for sorting date_sort_ascending = models.FloatField(blank=True, null=True) date_sort_descending = models.FloatField(blank=True, null=True) - publisher = models.TextField(null=True, blank=True, help_text="Enter multiple entities separated by a semicolon (;).") - languages = models.ManyToManyField(Language, help_text="Languages present in the manifest.", blank=True, null=True) + publisher = models.TextField( + null=True, + blank=True, + help_text="Enter multiple entities separated by a semicolon (;).", + ) + languages = models.ManyToManyField( + Language, help_text="Languages present in the manifest.", blank=True + ) attribution = models.CharField( max_length=255, null=True, blank=True, default="Emory University Libraries", - help_text="The institution holding the manifest" + help_text="The institution holding the manifest", ) logo = models.ImageField( - upload_to='logos/', - null=True, blank=True, + upload_to="logos/", + null=True, + blank=True, default="logos/lits-logo-web.png", - help_text="Upload the Logo of the institution holding the manifest." + help_text="Upload the Logo of the institution holding the manifest.", + ) + logo_url = models.URLField( + blank=True, + help_text="A URL in this field will take precedence over any logo file uploaded in the file field. Clear this field (if populated) to use an uploaded logo file instead.", ) license = models.CharField( max_length=255, null=True, blank=True, default="https://creativecommons.org/publicdomain/zero/1.0/", - help_text="Only enter a URI to a license statement." + help_text="Only enter a URI to a license statement.", ) scanned_by = models.CharField(max_length=255, null=True, blank=True) - identifier = models.CharField(max_length=255, null=True, blank=True, help_text="Call number or other unique id.") - identifier_uri = models.URLField(null=True, blank=True, help_text="Only enter a link to a catalog record.") - collections = models.ManyToManyField(Collection, blank=True, related_name='manifests') - pdf = models.URLField(null=True, blank=True, help_text="Enter a link to an online pdf.") - metadata = JSONField(default=dict, blank=True) - viewingdirection = models.CharField(max_length=13, choices=DIRECTIONS, default="left-to-right") + identifier = models.CharField( + max_length=255, + null=True, + blank=True, + help_text="Call number or other unique id.", + ) + identifier_uri = models.URLField( + null=True, blank=True, help_text="Only enter a link to a catalog record." + ) + collections = models.ManyToManyField( + Collection, blank=True, related_name="manifests" + ) + pdf = models.URLField( + null=True, blank=True, help_text="Enter a link to an online pdf." + ) + metadata = models.JSONField(default=dict, blank=True) + viewingdirection = models.CharField( + max_length=13, choices=DIRECTIONS, default="left-to-right" + ) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) - autocomplete_search_field = 'label' + autocomplete_search_field = "label" # TODO: This has to be removed/redone before we upgrade to Django 3 search_vector = SearchVectorField(null=True, editable=False) - image_server = models.ForeignKey(ImageServer, on_delete=models.DO_NOTHING, null=True) + image_server = models.ForeignKey( + ImageServer, on_delete=models.DO_NOTHING, null=True + ) # objects = ManifestManager() start_canvas = models.ForeignKey( - 'canvases.Canvas', + "canvases.Canvas", on_delete=models.SET_NULL, - related_name='start_canvas', + related_name="start_canvas", blank=True, - null=True + null=True, ) + searchable = models.BooleanField(default=True) def get_absolute_url(self): """Absolute URL for manifest @@ -209,7 +258,7 @@ def get_absolute_url(self): :return: Manifest's absolute URL :rtype: str """ - return '{h}/volume/{p}'.format(h=settings.HOSTNAME, p=self.pid) + return f"{settings.HOSTNAME}/volume/{self.pid}" def get_volume_url(self): """Convenience method for IIIF qualified URL. @@ -217,33 +266,35 @@ def get_volume_url(self): :return: IIIF manifest URL :rtype: str """ - return '{h}/volume/{p}/page/all'.format(h=settings.HOSTNAME, p=self.pid) + return f"{settings.HOSTNAME}/volume/{self.pid}/page/all" - class Meta: # pylint: disable = too-few-public-methods, missing-class-docstring - ordering = ['published_date'] + class Meta: # pylint: disable = too-few-public-methods, missing-class-docstring + ordering = ["published_date"] # indexes = [GinIndex(fields=['search_vector'])] @property - def publisher_bib(self): - """Concatenated property for bib citation. + def authors(self): + """Convert authors string into list - :rtype: str + :return: List of authors or ["[no author]"] if author field is empty + :rtype: list """ - return '{c} : {p}'.format(c=self.published_city, p=self.publisher) + if self.author: + return [s.strip() for s in self.author.split(";")] + return ["[no author]"] @property - def thumbnail_logo(self): - """Thumbnail of holding institution's logo + def publisher_bib(self): + """Concatenated property for bib citation. - :return: URL for logo thumbnail. :rtype: str """ - return '{h}/media/{l}'.format(h=settings.HOSTNAME, l=self.logo) + return f"{self.published_city} : {self.publisher}" @property def baseurl(self): """Convenience method to provide the base URL for a manifest.""" - return "%s/iiif/v2/%s" % (settings.HOSTNAME, self.pid) + return f"{settings.HOSTNAME}/iiif/v2/{self.pid}" @property def related_links(self): @@ -252,10 +303,58 @@ def related_links(self): :return: List of links related to Manifest :rtype: list """ - links = [link.link for link in self.relatedlink_set.all()] - links.append(self.get_volume_url()) + links = [ + ( + { + "@id": link.link, + "format": link.format, + } + if link.format + else link.link + ) + for link in self.relatedlink_set.all() + ] + links.append({"@id": self.get_volume_url(), "format": "text/html"}) return links + @property + def external_links(self): + """Dict of lists of external links for display on volume pages + + :return: Dict of external links ("related" and "seeAlso") + :rtype: dict + """ + # exclude internal links from related link set + related_links = self.relatedlink_set.exclude(link__icontains=settings.HOSTNAME) + # dict keys correspond to headings in sidebar + return { + "see_also": [ + link.link for link in related_links if link.is_structured_data + ], + "related": [ + link.link for link in related_links if not link.is_structured_data + ], + } + + @property + def see_also_links(self): + """List of links for IIIF v2 'seeAlso' field (structured data). + + :return: List of links to structured data describing Manifest + :rtype: list + """ + return [ + ( + { + "@id": link.link, + "format": link.format, + } + if link.format + else link.link + ) + for link in self.relatedlink_set.filter(is_structured_data=True) + ] + # TODO: Is this needed? It doesn't seem to be called anywhere. # Could we just use the label as is? def autocomplete_label(self): @@ -263,10 +362,10 @@ def autocomplete_label(self): :rtype: str """ - return self.label + return self.pid def __str__(self): - return self.label + return f"{self.pid} - title: {self.label}" # FIXME: This creates a circular dependency - Importing UserAnnotation here. # Furthermore, we shouldn't have any of the IIIF apps depend on Readux. Need @@ -275,34 +374,67 @@ def user_annotation_count(self, user=None): if user is None: return None from apps.readux.models import UserAnnotation - return UserAnnotation.objects.filter(owner=user).filter(canvas__manifest=self).count() - - #update search_vector every time the entry updates - def save(self, *args, **kwargs): # pylint: disable = arguments-differ - if not self._state.adding and 'pid' in self.get_dirty_fields() and self.image_server and self.image_server.storage_service == 's3': + return ( + UserAnnotation.objects.filter(owner=user) + .filter(canvas__manifest=self) + .count() + ) + + # update search_vector every time the entry updates + def save(self, *args, **kwargs): # pylint: disable = arguments-differ + + if ( + not self._state.adding + and "pid" in self.get_dirty_fields() + and self.image_server + and self.image_server.storage_service == "s3" + ): self.__rename_s3_objects() - super().save(*args, **kwargs) + # Remove the manifest from the index if searchable was changed to False. + # This only runs on update. Creating new manifests with searchable set + # to False throws an error because it has not been indexed yet. This + # is despite the search returning a result. + if self.searchable is False and self._state.adding is False: + from .documents import ManifestDocument + + response = ManifestDocument().search().query("match", pid=self.pid) + + if response.count() > 0: + ManifestDocument().update(self, action="delete") - Canvas = apps.get_model('canvases.canvas') try: - if self.start_canvas is None and hasattr(self, 'canvas_set') and self.canvas_set.exists(): - print([c.position] for c in self.canvas_set.all()) - self.start_canvas = self.canvas_set.all().order_by('position').first() - self.save() - except Canvas.DoesNotExist: + canvas_model = apps.get_model("canvases.canvas") + if ( + self.start_canvas is None + and hasattr(self, "canvas_set") + and self.canvas_set.exists() + ): + self.start_canvas = self.canvas_set.all().order_by("position").first() + canvas_model.objects.filter(manifest=self).update( + is_starting_page=False + ) + canvas_model.objects.filter(pk=self.start_canvas.id).update( + is_starting_page=True + ) + except canvas_model.DoesNotExist: self.start_canvas = None - # if 'update_fields' not in kwargs or 'search_vector' not in kwargs['update_fields']: - # instance = self._meta.default_manager.with_documents().get(pk=self.pk) - # instance.search_vector = instance.document - # instance.save(update_fields=['search_vector']) + super().save(*args, **kwargs) + + for collection in self.collections.all(): # pylint: disable = no-member + collection.modified_at = self.modified_at + collection.save() + if environ["DJANGO_ENV"] != "test": # pragma: no cover + index_manifest_task.apply_async(args=[str(self.id)]) + else: + index_manifest_task(str(self.id)) def delete(self, *args, **kwargs): """ - When a manifest is delted, the related canvas objects are deleted (`on_delete`=models.CASCADE). + When a manifest is deleted, the related canvas objects are deleted (`on_delete`=models.CASCADE). However, the `delete` method is not called on the canvas objects. We need to do that so the files can be cleaned up. https://docs.djangoproject.com/en/3.2/ref/models/fields/#django.db.models.CASCADE @@ -312,27 +444,50 @@ def delete(self, *args, **kwargs): super().delete(*args, **kwargs) + # if environ["DJANGO_ENV"] != 'test': # pragma: no cover + # de_index_manifest_task.apply_async(args=[str(self.id)]) + # else: + # de_index_manifest_task(str(self.id)) + def __rename_s3_objects(self): - original_pid = self.get_dirty_fields()['pid'] - keys = [f.key for f in self.image_server.bucket.objects.filter(Prefix=f'{original_pid}/')] + original_pid = self.get_dirty_fields()["pid"] + keys = [ + f.key + for f in self.image_server.bucket.objects.filter(Prefix=f"{original_pid}/") + ] for key in keys: obj = self.image_server.bucket.Object(key.replace(original_pid, self.pid)) - obj.copy({ 'Bucket': self.image_server.storage_path, 'Key': key }) + obj.copy({"Bucket": self.image_server.storage_path, "Key": key}) self.image_server.bucket.Object(key).delete() + # TODO: is this needed? class Note(models.Model): """Note for manifest""" + label = models.CharField(max_length=255) - language = models.CharField(max_length=10, default='en') + language = models.CharField(max_length=10, default="en") manifest = models.ForeignKey(Manifest, null=True, on_delete=models.CASCADE) + class RelatedLink(models.Model): - """ Links to related resources """ - id = models.UUIDField(primary_key=True, default=uuid4) + """Links to related resources""" + + id = models.UUIDField(primary_key=True, default=uuid4, editable=False) link = models.CharField(max_length=255) - data_type = models.CharField(max_length=255, default='Dataset') + data_type = models.CharField( + max_length=255, + default="Dataset", + ) + is_structured_data = models.BooleanField( + default=False, + help_text="True if this link is structured data that should appear in the manifest's " + + "'seeAlso' field; if false, the link will appear in the 'related' field instead. Leave " + + "unchecked if unsure.", + ) label = GenericRelation(ValueByLanguage) - format = models.CharField(max_length=255, choices=Choices.MIMETYPES, blank=True, null=True) + format = models.CharField( + max_length=255, choices=Choices.MIMETYPES, blank=True, null=True + ) profile = models.CharField(max_length=255, blank=True, null=True) manifest = models.ForeignKey(Manifest, on_delete=models.CASCADE) diff --git a/apps/iiif/manifests/services.py b/apps/iiif/manifests/services.py new file mode 100644 index 000000000..15a8ebb92 --- /dev/null +++ b/apps/iiif/manifests/services.py @@ -0,0 +1,326 @@ +"""Module of service classes and methods for ingest.""" + +import itertools +import re +import logging +from mimetypes import guess_type +from urllib.parse import unquote, urlparse + +from django.apps import apps +from tablib.core import Dataset + +from .models import Manifest, RelatedLink, Language + +LOGGER = logging.getLogger(__name__) + + +def clean_metadata(metadata): + """Normalize names of fields that align with Manifest fields. + + :param metadata: + :type metadata: tablib.Dataset + :return: Dictionary with keys matching Manifest fields + :rtype: dict + """ + fields = [f.name for f in Manifest._meta.get_fields()] + metadata = { + ( + key.casefold().replace(" ", "_") + if key.casefold().replace(" ", "_") in fields + else key + ): value + for key, value in metadata.items() + } + + for key in metadata.keys(): + if key != "metadata" and isinstance(metadata[key], list): + if isinstance(metadata[key][0], dict): + for meta_key in metadata[key][0].keys(): + if "value" in meta_key: + metadata[key] = metadata[key][0][meta_key] + else: + metadata[key] = ", ".join(metadata[key]) + + return metadata + + +def create_related_links(manifest, related_str): + """ + Create RelatedLink objects from supplied related links string and associate each with supplied + Manifest. String should consist of semicolon-separated URLs. + :param manifest: + :type manifest: iiif.manifest.models.Manifest + :param related_str: + :type related_str: str + :rtype: None + """ + for link in related_str.split(";"): + (link_format, _) = guess_type(link) + RelatedLink.objects.create( + manifest=manifest, + link=link, + format=link_format + or "text/html", # assume web page if MIME type cannot be determined + is_structured_data=False, # assume this is not meant for seeAlso + ) + + +def set_metadata(manifest, metadata): + """ + Update Manifest.metadata using supplied metadata dict + :param manifest: + :type manifest: iiif.manifest.models.Manifest + :param metadata: + :type metadata: dict + :rtype: None + """ + fields = [f.name for f in Manifest._meta.get_fields()] + for key, value in metadata.items(): + casefolded_key = key.casefold().replace(" ", "_") + if casefolded_key == "related": + # add RelatedLinks from metadata spreadsheet key "related" + create_related_links(manifest, value) + elif casefolded_key.startswith("language"): + for language in value.split(";"): + try: + manifest.languages.add(Language.objects.get(code=language)) + except Language.DoesNotExist: + LOGGER.warning(f"Language code {language} not found.") + elif casefolded_key in fields: + setattr(manifest, casefolded_key, value) + else: + # all other keys go into Manifest.metadata JSONField + if isinstance(manifest.metadata, list): + found_index = next( + ( + idx + for (idx, d) in enumerate(manifest.metadata) + if "label" in d and d["label"] == key + ), + None, + ) + if found_index: + # if value with this label exists, pop and re-insert + manifest.metadata.pop(found_index) + manifest.metadata.insert( + found_index, {"label": key, "value": value} + ) + else: + # if not, add label and value to end of list + manifest.metadata.append({"label": key, "value": value}) + elif isinstance(manifest.metadata, dict): + # convert to list of {label, value} as expected by iiif spec + manifest.metadata = [ + *[{"label": k, "value": v} for (k, v) in manifest.metadata.items()], + {"label": key, "value": value}, + ] + else: + # instantiate as list + manifest.metadata = [{"label": key, "value": value}] + manifest.save() + + +# def create_manifest(ingest): +# """ +# Create or update a Manifest from supplied metadata and images. +# :return: New or updated Manifest with supplied `pid` +# :rtype: iiif.manifest.models.Manifest +# """ +# manifest = None +# # Make a copy of the metadata so we don't extract it over and over. +# try: +# if not bool(ingest.manifest) or ingest.manifest is None: +# ingest.open_metadata() + +# metadata = dict(ingest.metadata) +# except TypeError: +# metadata = None +# if metadata: +# if "pid" in metadata: +# manifest, _ = Manifest.objects.get_or_create( +# pid=metadata["pid"].replace("_", "-") +# ) +# else: +# manifest = Manifest.objects.create() +# set_metadata(manifest, metadata) +# else: +# manifest = Manifest() + +# manifest.image_server = ingest.image_server + +# # This was giving me a 'django.core.exceptions.AppRegistryNotReady: Models aren't loaded yet' error. +# Remote = apps.get_model("ingest.remote") + +# # Ensure that manifest has an ID before updating the M2M relationship +# manifest.save() +# if not isinstance(ingest, Remote): +# manifest.refresh_from_db() +# manifest.collections.set(ingest.collections.all()) +# # Save again once relationship is set +# manifest.save() +# else: +# RelatedLink( +# manifest=manifest, +# link=ingest.remote_url, +# format="application/ld+json", +# is_structured_data=True, +# ).save() + +# return manifest + + +def extract_image_server(canvas): + """Determines the IIIF image server URL for a given IIIF Canvas + + :param canvas: IIIF Canvas + :type canvas: dict + :return: IIIF image server URL + :rtype: str + """ + url = urlparse(canvas["images"][0]["resource"]["service"]["@id"]) + parts = url.path.split("/") + parts.pop() + base_path = "/".join(parts) + host = url.hostname + if url.port is not None: + host = "{h}:{p}".format(h=url.hostname, p=url.port) + return "{s}://{h}{p}".format(s=url.scheme, h=host, p=base_path) + + +def parse_iiif_v2_manifest(data): + """Parse IIIF Manifest based on v2.1.1 or the presentation API. + https://iiif.io/api/presentation/2.1 + + :param data: IIIF Presentation v2.1.1 manifest + :type data: dict + :return: Extracted metadata + :rtype: dict + """ + properties = {} + manifest_data = [] + + if "metadata" in data: + manifest_data.append({"metadata": data["metadata"]}) + + for iiif_metadata in [ + {prop["label"]: prop["value"]} for prop in data["metadata"] + ]: + properties.update(iiif_metadata) + + # Sometimes, the label appears as a list. + if "label" in data.keys() and isinstance(data["label"], list): + data["label"] = " ".join(data["label"]) + + manifest_data.extend( + [{prop: data[prop]} for prop in data if isinstance(data[prop], str)] + ) + + for datum in manifest_data: + properties.update(datum) + + uri = urlparse(data["@id"]) + + if not uri.query: + properties["pid"] = uri.path.split("/")[-2] + else: + properties["pid"] = uri.query + + if "description" in data.keys(): + if isinstance(data["description"], list): + if isinstance(data["description"][0], dict): + en = [ + lang["@value"] + for lang in data["description"] + if lang["@language"] == "en" + ] + properties["summary"] = ( + data["description"][0]["@value"] if not en else en[0] + ) + else: + properties["summary"] = data["description"][0] + else: + properties["summary"] = data["description"] + + if "logo" in properties: + properties["logo_url"] = properties["logo"] + properties.pop("logo") + + manifest_metadata = clean_metadata(properties) + + return manifest_metadata + + +def parse_iiif_v2_canvas(canvas): + """ """ + canvas_id = canvas["@id"].split("/") + pid = canvas_id[-1] if canvas_id[-1] != "canvas" else canvas_id[-2] + + service = urlparse(canvas["images"][0]["resource"]["service"]["@id"]) + resource = unquote(service.path.split("/").pop()) + + summary = canvas["description"] if "description" in canvas.keys() else "" + label = canvas["label"] if "label" in canvas.keys() else "" + return { + "pid": pid, + "height": canvas["height"], + "width": canvas["width"], + "summary": summary, + "label": label, + "resource": resource, + } + + +def get_metadata_from(files): + """ + Find metadata file in uploaded files. + :return: If metadata file exists, returns the values. If no file, returns None. + :rtype: list or None + """ + metadata = None + for file in files: + if metadata is not None: + continue + if "zip" in guess_type(file.name)[0]: + continue + if "metadata" in file.name.casefold(): + stream = file.read() + if ( + "csv" in guess_type(file.name)[0] + or "tab-separated" in guess_type(file.name)[0] + ): + metadata = Dataset().load(stream.decode("utf-8-sig"), format="csv").dict + else: + metadata = Dataset().load(stream).dict + return metadata + + +def get_associated_meta(all_metadata, file): + """ + Associate metadata with filename. + :return: If a matching filename is found, returns the row as dict, + with generated pid. Otherwise, returns {}. + :rtype: dict + """ + file_meta = {} + extless_filename = file.name[0 : file.name.rindex(".")] + for meta_dict in all_metadata: + metadata_found_filename = None + for key, val in meta_dict.items(): + if key.casefold() == "filename": + metadata_found_filename = val + # Match filename column, case-sensitive, against filename + if metadata_found_filename and metadata_found_filename in ( + extless_filename, + file.name, + ): + file_meta = meta_dict + return file_meta + + +def normalize_header(iterator): + """Normalize the header row of a metadata CSV""" + # ignore unicode characters and strip whitespace + header_row = next(iterator).encode("ascii", "ignore").decode().strip() + # lowercase the word "pid" in this row so we can access it easily + header_row = re.sub(r"[Pp][Ii][Dd]", lambda m: m.group(0).casefold(), header_row) + return itertools.chain([header_row], iterator) diff --git a/apps/iiif/manifests/tasks.py b/apps/iiif/manifests/tasks.py new file mode 100644 index 000000000..b149050b8 --- /dev/null +++ b/apps/iiif/manifests/tasks.py @@ -0,0 +1,35 @@ +from celery import Celery +from django.conf import settings + +app = Celery('apps.iiif.manifests', result_extended=True) +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) + +@app.task(name='index_manifest', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) +def index_manifest_task(manifest_id): + """Background task index Manifest after save. + + :param manifest_id: Primary key for .models.Manifest object + :type manifest_id: UUID + + """ + from .models import Manifest + from .documents import ManifestDocument + index = ManifestDocument() + manifest = Manifest.objects.get(pk=manifest_id) + index.update(manifest, True, 'index') + + +@app.task(name='de-index_manifest', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) +def de_index_manifest_task(manifest_id): + """Background task to de-index Manifest after delete. + + :param manifest_id: Primary key for .models.Manifest object + :type manifest_id: UUID + + """ + from .models import Manifest + from .documents import ManifestDocument + index = ManifestDocument() + manifest = Manifest.objects.get(pk=manifest_id) + index.update(manifest, True, 'delete') diff --git a/apps/iiif/manifests/templates/admin/change_list_override.html b/apps/iiif/manifests/templates/admin/change_list_override.html new file mode 100644 index 000000000..1f76d59dc --- /dev/null +++ b/apps/iiif/manifests/templates/admin/change_list_override.html @@ -0,0 +1,26 @@ +{% extends "admin/change_list.html" %} + +{% load i18n %} + +{# adapted from django-import-export #} +{% block object-tools %} +Really smart annotation
", + "creator": {"id": user.username, "name": user.name}, + "modified": "2022-06-06T20:29:08.108Z", + } + ], + "target": { + "source": canvas.resource_id, + "selector": { + "type": "FragmentSelector", + "conformsTo": "http://www.w3.org/TR/media-frags/", + "value": "xywh=pixel:575.7898559570312,1263.1917724609375,677.0464477539062,737.7884521484375", + }, + }, + "@context": "http://www.w3.org/ns/anno.jsonld", + "id": "#51602663-36ee-4692-a327-2f438daf48a9", + } + + deserialized_annotation, _ = deserialize("annotation", web_annotation) + annotation = UserAnnotation(**deserialized_annotation) + assert annotation.owner == user + assert annotation.canvas == canvas + assert annotation.content == web_annotation["body"][0]["value"] + assert annotation.x == 575.7898559570312 + assert annotation.y == 1263.1917724609375 + assert annotation.w == 677.0464477539062 + assert annotation.h == 737.7884521484375 + + def test_web_annotation_comment_svg_deserialization(self): + user = UserFactory.create() + canvas = CanvasFactory.create(manifest=ManifestFactory.create()) + web_annotation = { + "type": "Annotation", + "motivation": "commenting", + "body": [ + { + "purpose": "commenting", + "type": "TextualBody", + "created": "2022-06-06T20:38:37.123Z", + "value": "Even smarter annotation
", + "creator": {"id": user.username, "name": user.name}, + "modified": "2022-06-06T20:38:37.123Z", + }, + { + "type": "TextualBody", + "value": "Tag One", + "purpose": "tagging", + "creator": {"id": user.username, "name": user.name}, + "created": "2022-06-06T20:38:43.052Z", + "modified": "2022-06-06T20:38:43.720Z", + }, + ], + "target": { + "source": canvas.resource_id, + "selector": { + "type": "SvgSelector", + "value": '', + "refinedBy": { + "type": "FragmentSelector", + "value": "xywh=448.96453857421875,1384.190185546875,1224.1728515625,1224.173095703125", + }, + }, + }, + "@context": "http://www.w3.org/ns/anno.jsonld", + "id": "#db7cd136-cdb7-4b1e-b33c-f0afd66c1aee", + } + + deserialized_annotation, tags = deserialize("annotation", web_annotation) + annotation = UserAnnotation(**deserialized_annotation) + annotation.save() + for tag in tags: + annotation.tags.add(tag) + assert annotation.owner == user + assert annotation.canvas == canvas + assert annotation.content == web_annotation["body"][0]["value"] + assert "Tag One" in annotation.tag_list + assert annotation.svg + assert annotation.x == 448.96453857421875 + assert annotation.y == 1384.190185546875 + assert annotation.w == 1224.1728515625 + assert annotation.h == 1224.173095703125 + + def test_web_annotation_comment_range_deserialization(self): + user = UserFactory.create() + canvas = CanvasFactory.create(manifest=ManifestFactory.create()) + start = AnnotationFactory.create(canvas=canvas, order=3) + end = AnnotationFactory.create(canvas=canvas, order=13) + web_annotation = { + "type": "Annotation", + "motivation": "commenting", + "body": [ + { + "type": "TextualBody", + "value": "Yet another annotation
", + "purpose": "commenting", + "creator": {"id": user.username, "name": user.name}, + } + ], + "target": { + "source": canvas.resource_id, + "selector": { + "type": "RangeSelector", + "startSelector": { + "@type": "XPathSelector", + "value": f"//*[@id='{start.pk}']", + "refinedBy": {"@type": "TextPositionSelector", "start": 1}, + }, + "endSelector": { + "type": "XPathSelector", + "value": f"//*[@id='{end.pk}']", + "refinedBy": {"@type": "TextPositionSelector", "end": 5}, + }, + }, + }, + "@context": "http://www.w3.org/ns/anno.jsonld", + "id": "#da324841-beaa-4710-858e-f128580c6f2d", + } + deserialized_annotation, tags = deserialize("annotation", web_annotation) + annotation = UserAnnotation(**deserialized_annotation) + annotation.save() + for tag in tags: + annotation.tags.add(tag) + assert annotation.owner == user + assert annotation.canvas == canvas + assert annotation.content == web_annotation["body"][0]["value"] + assert annotation.start_selector == start + assert ( + annotation.start_offset + == web_annotation["target"]["selector"]["startSelector"]["refinedBy"][ + "start" + ] + ) + assert annotation.end_selector == end + assert ( + annotation.end_offset + == web_annotation["target"]["selector"]["endSelector"]["refinedBy"]["end"] + ) diff --git a/apps/iiif/serializers/user_annotation_list.py b/apps/iiif/serializers/user_annotation_list.py index 6073726b8..2bb435aba 100644 --- a/apps/iiif/serializers/user_annotation_list.py +++ b/apps/iiif/serializers/user_annotation_list.py @@ -3,42 +3,47 @@ import json from django.core.serializers.base import SerializerDoesNotExist from django.core.serializers import serialize -from apps.iiif.serializers.annotation_list import Serializer as IIIFAnnotationListSerializer +from apps.iiif.serializers.annotation_list import ( + Serializer as IIIFAnnotationListSerializer, +) import config.settings.local as settings + class Serializer(IIIFAnnotationListSerializer): """ IIIF V2 Annotation List https://iiif.io/api/presentation/2.1/#annotation-list """ def get_dump_object(self, obj): - if ((self.version == 'v2') or (self.version is None)): + if (self.version == "v2") or (self.version is None): data = { "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": '{h}/annotations/{u}/{m}/list/{c}'.format( + "@id": "{h}/annotations/{u}/{m}/list/{c}".format( h=settings.HOSTNAME, u=self.owners[0].username, m=obj.manifest.pid, - c=obj.pid + c=obj.pid, ), "@type": "sc:AnnotationList", "resources": json.loads( serialize( - 'annotation', - obj.userannotation_set.filter( - owner__in=[self.owners[0].id] - ), - is_list=True + "annotation", + obj.userannotation_set.filter(owner__in=[self.owners[0].id]), + is_list=True, ) - ) + ), } return data return None + class Deserializer: """Deserialize IIIF Annotation List :raises SerializerDoesNotExist: Not yet implemented. """ + def __init__(self, *args, **kwargs): - raise SerializerDoesNotExist("user_annotation_list is a serialization-only serializer") + raise SerializerDoesNotExist( + "user_annotation_list is a serialization-only serializer" + ) diff --git a/apps/iiif/serializers/v2/annotation.py b/apps/iiif/serializers/v2/annotation.py new file mode 100644 index 000000000..e880a531b --- /dev/null +++ b/apps/iiif/serializers/v2/annotation.py @@ -0,0 +1,101 @@ +"""Module for serializing IIIF V2 Annotation Lists""" + +import json +from re import findall +from bs4 import BeautifulSoup +from django.core.serializers import deserialize +from django.contrib.auth import get_user_model +from apps.iiif.serializers.annotation import Serializer as AnnotationSerializer +from apps.iiif.annotations.models import Annotation +from apps.iiif.annotations.choices import AnnotationPurpose, AnnotationSelector +from apps.iiif.canvases.models import Canvas + +USER = get_user_model() + + +class Serializer(AnnotationSerializer): + """Convert a queryset to IIIF Annotation""" + + +def Deserializer(data): # pylint: disable=invalid-name + """Deserialize V2 Annotation. + + Args: + data (dict): V2 IIIF Annotation + + Returns: + annotation: annotation dict + """ + + if isinstance(data, str): + data = json.loads(data) + + annotation = { + "id": data["@id"], + "owner": USER.objects.get(name=data["annotatedBy"]["name"]), + } + tags = [] + + if data["motivation"] == "oa:commenting": + annotation["motivation"] = Annotation.OA_COMMENTING + annotation["purpose"] = AnnotationPurpose("CM") + + if data["motivation"] == "oa:painting": + annotation["motivation"] = Annotation.SC_PAINTING + annotation["purpose"] = AnnotationPurpose("SP") + + annotation["primary_selector"] = AnnotationSelector("FR") + source_parts = data["on"]["full"].split("/") + canvas_pid = source_parts[-1] if source_parts[-1] != "canvas" else source_parts[-2] + annotation["canvas"] = Canvas.objects.get(pid=canvas_pid) + + resources = ( + data["resource"] if isinstance(data["resource"], list) else [data["resource"]] + ) + + for resource in resources: + if resource["@type"] == "cnt:ContentAsText": + annotation["resource_type"] = Annotation.OCR + if resource["@type"] == "dctypes:Text": + annotation["resource_type"] = Annotation.TEXT + + if resource["@type"] == "oa:Tag": + tags.append(resource["chars"]) + else: + annotation["content"] = resource["chars"] + soup = BeautifulSoup(resource["chars"], "html.parser") + annotation["raw_content"] = soup.get_text(separator=" ", strip=True) + + annotation["x"], annotation["y"], annotation["w"], annotation["h"] = [ + float(n) for n in data["on"]["selector"]["value"].split("=")[-1].split(",") + ] + + if data["on"]["selector"]["item"]["@type"] == "oa:SvgSelector": + annotation["svg"] = data["on"]["selector"]["item"]["value"] + + if data["on"]["selector"]["item"]["@type"] == "RangeSelector": + annotation["primary_selector"] = AnnotationSelector("RG") + annotation["start_selector"], _ = Annotation.objects.get_or_create( + pk=findall( + r"([A-Za-z0-9\-]+)", + data["on"]["selector"]["item"]["startSelector"]["value"], + )[-1] + ) + annotation["end_selector"], _ = Annotation.objects.get_or_create( + pk=findall( + r"([A-Za-z0-9\-]+)", + data["on"]["selector"]["item"]["endSelector"]["value"], + )[-1] + ) + annotation["start_offset"] = data["on"]["selector"]["item"]["startSelector"][ + "refinedBy" + ]["start"] + + annotation["end_offset"] = data["on"]["selector"]["item"]["endSelector"][ + "refinedBy" + ]["end"] + + return ( + annotation, + tags, + ) diff --git a/apps/iiif/serializers/v2/canvas.py b/apps/iiif/serializers/v2/canvas.py new file mode 100644 index 000000000..41eb3d5ed --- /dev/null +++ b/apps/iiif/serializers/v2/canvas.py @@ -0,0 +1,26 @@ +"""Module for serializing IIIF V2 Canvas Lists""" + +import re +from apps.iiif.serializers.canvas import Serializer as CanvasSerializer +from apps.iiif.manifests.models import Manifest + + +class Serializer(CanvasSerializer): + """Convert a queryset to IIIF Canvas""" + + +def Deserializer(data): # pylint: disable=invalid-name + """Deserialize V2 Canvas. + + Args: + data (dict): V2 IIIF Canvas + + Returns: + dict: canvas dict + """ + return { + "pid": data["@id"].split("/")[-1], + "resource": data["images"][0]["resource"]["service"]["@id"].split("/")[-1], + "width": data["images"][0]["resource"]["width"], + "height": data["images"][0]["resource"]["height"], + } diff --git a/apps/iiif/serializers/v2/fixtures/v2_manifest.json b/apps/iiif/serializers/v2/fixtures/v2_manifest.json new file mode 100644 index 000000000..59dd53d4e --- /dev/null +++ b/apps/iiif/serializers/v2/fixtures/v2_manifest.json @@ -0,0 +1,120 @@ +{ + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest", + "label": "CSM Entry Checklist (S/N 1001)", + "metadata": [ + { + "label": "Author", + "value": "NASA" + }, + { + "label": "Publisher", + "value": "NASA" + }, + { + "label": "Place of Publication", + "value": "Place of publication" + }, + { + "label": "Publication Date", + "value": "1971" + }, + { + "label": "Notes", + "value": null + }, + { + "label": "Record Created", + "value": "2020-05-26T18:15:14.606Z" + }, + { + "label": "Edition Type", + "value": "Readux IIIF Exported Edition" + }, + { + "label": "About Readux", + "value": "https://readux.ecdsdev.org/about/" + }, + { + "label": "Annotators", + "value": "Jay Varner" + }, + { + "label": "Export Date", + "value": "2025-07-03T13:32:36.782" + } + ], + "description": "CSM Entry Checklist (S/N 1001) from the Apollo 15 Flight Data File (FDF). \"There are two CSM Launch Checklists, two LM Activation Checklists, and two CSM Entry Checklists - one each for the CDR [commander] and for either the CMP [command module pilot] or LMP [lunar module pilot]. We each had one to expedite the timeline and not need to pass a checklist back and forth.\" - Col. Scott", + "related": ["https://readux.io/volume/spb1b/page/all"], + "within": ["https://readux.io/collection/apollo-15"], + "thumbnail": { + "@id": "https://iip.readux.io/iiif/2/spb1b_000.tif/full/600,/0/default.jpg", + "service": { + "@context": "http://iiif.io/api/image/2/context.json", + "@id": "https://iip.readux.io/iiif/2/spb1b_000.tif", + "profile": "http://iiif.io/api/image/2/level1.json" + } + }, + "attribution": "Emory University Libraries", + "logo": "https://readux.io/media/logos/lits-logo-web.png", + "license": "https://creativecommons.org/publicdomain/zero/1.0/", + "viewingDirection": "left-to-right", + "viewingHint": "paged", + "sequences": [ + { + "@id": "https://readux.io/iiif/v2/spb1b/sequence/normal", + "@type": "sc:Sequence", + "label": "Current Page Order", + "startCanvas": "https://readux.io/iiif/spb1b/canvas/spb1b_000.tif", + "canvases": [ + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "https://readux.io/iiif/spb1b/canvas/spb1b_000.tif", + "@type": "sc:Canvas", + "label": "1", + "height": 3600, + "width": 2814, + "images": [ + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "https://readux.io/iiif/spb1b/annotation/spb1b_000.tif", + "@type": "oa:Annotation", + "motivation": "sc:painting", + "resource": { + "@id": "https://iip.readux.io/iiif/2/spb1b_000.tif/full/full/0/default.jpg", + "@type": "dctypes:Image", + "format": "image/jpeg", + "height": 3600, + "width": 2814, + "service": { + "@context": "https://iiif.io/api/image/2/context.json", + "@id": "https://iip.readux.io/iiif/2/spb1b_000.tif", + "profile": "https://iiif.io/api/image/2/level2.json" + } + }, + "on": "https://readux.io/iiif/spb1b/canvas/spb1b_000.tif" + } + ], + "thumbnail": { + "@id": "https://iip.readux.io/iiif/2/spb1b_000.tif/full/200,/0/default.jpg", + "height": 250, + "width": 200 + }, + "otherContent": [ + { + "@id": "https://readux.io/iiif/v2/spb1b/list/spb1b_000.tif", + "@type": "sc:AnnotationList", + "label": "OCR Text" + }, + { + "label": "Annotations by jay", + "@type": "sc:AnnotationList", + "@id": "https://readux.io/annotations/jay/spb1b/list/spb1b_000.tif" + } + ] + } + ] + } + ] +} diff --git a/apps/iiif/serializers/v2/fixtures/v2_ocr.json b/apps/iiif/serializers/v2/fixtures/v2_ocr.json new file mode 100644 index 000000000..deee6fec7 --- /dev/null +++ b/apps/iiif/serializers/v2/fixtures/v2_ocr.json @@ -0,0 +1,449 @@ +{ + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "https://readux.io/iiif/v2/spb1b/list/spb1b_000.tif", + "@type": "sc:AnnotationList", + "resources": [ + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "fc3320b8-951f-414f-8bf2-dca64555c8be", + "@type": "oa:Annotation", + "motivation": "sc:painting", + "annotatedBy": { + "name": "OCR" + }, + "resource": { + "@type": "cnt:ContentAsText", + "format": "text/html", + "chars": "APOLLO", + "language": "en" + }, + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=1256,806,340,69", + "item": { + "@type": "oa:FragmentSelector" + } + } + }, + "stylesheet": { + "type": "CssStylesheet", + "value": ".anno-fc3320b8-951f-414f-8bf2-dca64555c8be: { height: 69px; width: 340px; font-size: 43.125px; }" + } + }, + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "e90bfe36-572b-47d7-8743-4d881d82ee3c", + "@type": "oa:Annotation", + "motivation": "sc:painting", + "annotatedBy": { + "name": "OCR" + }, + "resource": { + "@type": "cnt:ContentAsText", + "format": "text/html", + "chars": "15", + "language": "en" + }, + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=1661,808,36,67", + "item": { + "@type": "oa:FragmentSelector" + } + } + }, + "stylesheet": { + "type": "CssStylesheet", + "value": ".anno-e90bfe36-572b-47d7-8743-4d881d82ee3c: { height: 67px; width: 36px; font-size: 41.875px; }" + } + }, + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "5a1b56bb-b32d-419a-969a-0465982c8a0f", + "@type": "oa:Annotation", + "motivation": "sc:painting", + "annotatedBy": { + "name": "OCR" + }, + "resource": { + "@type": "cnt:ContentAsText", + "format": "text/html", + "chars": "CSM", + "language": "en" + }, + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=1069,982,229,85", + "item": { + "@type": "oa:FragmentSelector" + } + } + }, + "stylesheet": { + "type": "CssStylesheet", + "value": ".anno-5a1b56bb-b32d-419a-969a-0465982c8a0f: { height: 85px; width: 229px; font-size: 53.125px; }" + } + }, + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "1c112e0a-f070-44ac-b49b-4f037ee7d2b9", + "@type": "oa:Annotation", + "motivation": "sc:painting", + "annotatedBy": { + "name": "OCR" + }, + "resource": { + "@type": "cnt:ContentAsText", + "format": "text/html", + "chars": "ENTRY", + "language": "en" + }, + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=1461,984,384,87", + "item": { + "@type": "oa:FragmentSelector" + } + } + }, + "stylesheet": { + "type": "CssStylesheet", + "value": ".anno-1c112e0a-f070-44ac-b49b-4f037ee7d2b9: { height: 87px; width: 384px; font-size: 54.375px; }" + } + }, + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "ece5a9ff-ee53-4106-8177-5b18fe3e1da8", + "@type": "oa:Annotation", + "motivation": "sc:painting", + "annotatedBy": { + "name": "OCR" + }, + "resource": { + "@type": "cnt:ContentAsText", + "format": "text/html", + "chars": "CHECKLIST", + "language": "en" + }, + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=1164,1108,650,94", + "item": { + "@type": "oa:FragmentSelector" + } + } + }, + "stylesheet": { + "type": "CssStylesheet", + "value": ".anno-ece5a9ff-ee53-4106-8177-5b18fe3e1da8: { height: 94px; width: 650px; font-size: 58.75px; }" + } + }, + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "2e31ae5a-d662-401b-9385-9eed5e370cfe", + "@type": "oa:Annotation", + "motivation": "sc:painting", + "annotatedBy": { + "name": "OCR" + }, + "resource": { + "@type": "cnt:ContentAsText", + "format": "text/html", + "chars": "PART", + "language": "en" + }, + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=1038,1316,290,71", + "item": { + "@type": "oa:FragmentSelector" + } + } + }, + "stylesheet": { + "type": "CssStylesheet", + "value": ".anno-2e31ae5a-d662-401b-9385-9eed5e370cfe: { height: 71px; width: 290px; font-size: 44.375px; }" + } + }, + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "d26b5984-e2d0-40ed-99f4-069ca8e8d2c3", + "@type": "oa:Annotation", + "motivation": "sc:painting", + "annotatedBy": { + "name": "OCR" + }, + "resource": { + "@type": "cnt:ContentAsText", + "format": "text/html", + "chars": "NO", + "language": "en" + }, + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=1367,1319,136,70", + "item": { + "@type": "oa:FragmentSelector" + } + } + }, + "stylesheet": { + "type": "CssStylesheet", + "value": ".anno-d26b5984-e2d0-40ed-99f4-069ca8e8d2c3: { height: 70px; width: 136px; font-size: 43.75px; }" + } + }, + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "5263432e-ea5e-433a-a3b3-7346d7b85a54", + "@type": "oa:Annotation", + "motivation": "sc:painting", + "annotatedBy": { + "name": "OCR" + }, + "resource": { + "@type": "cnt:ContentAsText", + "format": "text/html", + "chars": ".", + "language": "en" + }, + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=1512,1320,18,68", + "item": { + "@type": "oa:FragmentSelector" + } + } + }, + "stylesheet": { + "type": "CssStylesheet", + "value": ".anno-5263432e-ea5e-433a-a3b3-7346d7b85a54: { height: 68px; width: 18px; font-size: 42.5px; }" + } + }, + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "59f98d81-e33e-4502-8b10-dec7ca4d36a9", + "@type": "oa:Annotation", + "motivation": "sc:painting", + "annotatedBy": { + "name": "OCR" + }, + "resource": { + "@type": "cnt:ContentAsText", + "format": "text/html", + "chars": "S", + "language": "en" + }, + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=2033,1322,19,67", + "item": { + "@type": "oa:FragmentSelector" + } + } + }, + "stylesheet": { + "type": "CssStylesheet", + "value": ".anno-59f98d81-e33e-4502-8b10-dec7ca4d36a9: { height: 67px; width: 19px; font-size: 41.875px; }" + } + }, + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "2e22fc7e-b99c-4af7-9a8b-79f069a2dc3b", + "@type": "oa:Annotation", + "motivation": "sc:painting", + "annotatedBy": { + "name": "OCR" + }, + "resource": { + "@type": "cnt:ContentAsText", + "format": "text/html", + "chars": "/", + "language": "en" + }, + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=2088,1323,19,67", + "item": { + "@type": "oa:FragmentSelector" + } + } + }, + "stylesheet": { + "type": "CssStylesheet", + "value": ".anno-2e22fc7e-b99c-4af7-9a8b-79f069a2dc3b: { height: 67px; width: 19px; font-size: 41.875px; }" + } + }, + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "cdd6c9d9-f542-45c9-8613-bf4844109f28", + "@type": "oa:Annotation", + "motivation": "sc:painting", + "annotatedBy": { + "name": "OCR" + }, + "resource": { + "@type": "cnt:ContentAsText", + "format": "text/html", + "chars": "N", + "language": "en" + }, + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=2135,1323,19,67", + "item": { + "@type": "oa:FragmentSelector" + } + } + }, + "stylesheet": { + "type": "CssStylesheet", + "value": ".anno-cdd6c9d9-f542-45c9-8613-bf4844109f28: { height: 67px; width: 19px; font-size: 41.875px; }" + } + }, + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "0ceb5275-3ad8-4a69-80d6-378a50308f7c", + "@type": "oa:Annotation", + "motivation": "sc:painting", + "annotatedBy": { + "name": "OCR" + }, + "resource": { + "@type": "cnt:ContentAsText", + "format": "text/html", + "chars": "SKB32100115-305", + "language": "en" + }, + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=877,1470,733,69", + "item": { + "@type": "oa:FragmentSelector" + } + } + }, + "stylesheet": { + "type": "CssStylesheet", + "value": ".anno-0ceb5275-3ad8-4a69-80d6-378a50308f7c: { height: 69px; width: 733px; font-size: 43.125px; }" + } + }, + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "0754507c-562d-4b5e-9046-88227d6c0b7d", + "@type": "oa:Annotation", + "motivation": "sc:painting", + "annotatedBy": { + "name": "OCR" + }, + "resource": { + "@type": "cnt:ContentAsText", + "format": "text/html", + "chars": "1001", + "language": "en" + }, + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=2005,1476,193,67", + "item": { + "@type": "oa:FragmentSelector" + } + } + }, + "stylesheet": { + "type": "CssStylesheet", + "value": ".anno-0754507c-562d-4b5e-9046-88227d6c0b7d: { height: 67px; width: 193px; font-size: 41.875px; }" + } + } + ] +} diff --git a/apps/iiif/serializers/v2/fixtures/v2_user_annotation_list.json b/apps/iiif/serializers/v2/fixtures/v2_user_annotation_list.json new file mode 100644 index 000000000..197e3a4b2 --- /dev/null +++ b/apps/iiif/serializers/v2/fixtures/v2_user_annotation_list.json @@ -0,0 +1,98 @@ +{ + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "https://readux.io/annotations/jay/spb1b/list/spb1b_000.tif", + "@type": "sc:AnnotationList", + "resources": [ + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "28e92e3c-0407-482e-98d9-39ecc1ee0070", + "@type": "oa:Annotation", + "motivation": "oa:commenting", + "annotatedBy": { + "name": "Dominique Wilkins" + }, + "resource": { + "@type": "dctypes:Text", + "format": "text/html", + "chars": "some annotation
", + "language": "en" + }, + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=1069,984,776,87", + "item": { + "@type": "RangeSelector", + "startSelector": { + "@type": "XPathSelector", + "value": "//*[@id='5a1b56bb-b32d-419a-969a-0465982c8a0f']", + "refinedBy": { + "@type": "TextPositionSelector", + "start": 0 + } + }, + "endSelector": { + "@type": "XPathSelector", + "value": "//*[@id='ece5a9ff-ee53-4106-8177-5b18fe3e1da8']", + "refinedBy": { + "@type": "TextPositionSelector", + "end": 2 + } + } + } + } + }, + "stylesheet": { + "type": "CssStylesheet", + "value": ".anno-28e92e3c-0407-482e-98d9-39ecc1ee0070 { background: rgba(0, 191, 255, 0.5); }" + } + }, + { + "@context": "http://iiif.io/api/presentation/2/context.json", + "@id": "6c1c7843-832b-46b3-aca9-da957b07be6d", + "@type": "oa:Annotation", + "motivation": ["oa:commenting", "oa:tagging"], + "annotatedBy": { + "name": "Dominique Wilkins" + }, + "resource": [ + { + "@type": "dctypes:Text", + "format": "text/html", + "chars": "box
", + "language": "en" + }, + { + "@type": "oa:Tag", + "chars": "example" + } + ], + "on": { + "full": "https://readux.io/iiif/v2/spb1b/canvas/spb1b_000.tif", + "@type": "oa:SpecificResource", + "within": { + "@id": "https://readux.io/iiif/v2/spb1b/manifest", + "@type": "sc:Manifest" + }, + "selector": { + "@type": "oa:FragmentSelector", + "value": "xywh=813,1920,527,553", + "item": { + "@type": "oa:Choice", + "value": "", + "default": { + "@type": "oa:FragmentSelector", + "value": "xywh=813,1920,527,553" + } + } + } + } + } + ] +} diff --git a/apps/iiif/serializers/v2/manifest.py b/apps/iiif/serializers/v2/manifest.py new file mode 100644 index 000000000..d6c3ab393 --- /dev/null +++ b/apps/iiif/serializers/v2/manifest.py @@ -0,0 +1,69 @@ +"""Module for serializing IIIF V2 Manifest Lists""" + +import re +from apps.iiif.serializers.manifest import Serializer as ManifestSerializer +from apps.iiif.manifests.models import Manifest + + +class Serializer(ManifestSerializer): + """Convert a queryset to IIIF Manifest""" + + +def parse_fields(attribute): + """Get attributes from V2 Manifest + + Args: + attribute (tuple): _description_ + """ + key, value = attribute + fields = [field.name for field in Manifest._meta.get_fields()] + field = re.sub(r"\([^)]*\)", "", key).strip() + field = field.replace(" ", "_") + field = field.lower() + if field == "publication_date": + return {"published_date": value} + if field == "place_of_publication": + return {"published_city": value} + if field in fields: + return {field: value} + return None + + +def Deserializer(data): # pylint: disable=invalid-name + """Deserialize V2 Manifest. + + Args: + data (dict): V2 IIIF Manifest + + Returns: + tuple: manifest dict and relationships + """ + relations = { + "collections": data["within"], + "related_links": data["seeAlso"], + "canvases": [ + canvas["@id"].split("/")[-1] for canvas in data["sequences"][0]["canvases"] + ], + } + + manifest = { + "pid": data["@id"].split("/")[-2], + "summary": data["description"], + } + + try: + metadata = data.pop("metadata") + for metadatum in metadata: + attribute = parse_fields((metadatum["label"], metadatum["value"])) + if attribute is not None: + manifest = {**manifest, **attribute} + except KeyError: + # Maybe no metadata + pass + + for key, value in data.items(): + attribute = parse_fields((key, value)) + if attribute is not None: + data = {**data, **attribute} + + return (manifest, relations) diff --git a/apps/iiif/serializers/v2/tests/test_annotation.py b/apps/iiif/serializers/v2/tests/test_annotation.py new file mode 100644 index 000000000..cd1ec9cfa --- /dev/null +++ b/apps/iiif/serializers/v2/tests/test_annotation.py @@ -0,0 +1,222 @@ +"""Test Module for IIIF Serializers""" + +import os +import json +from django.test import TestCase +from django.core.serializers import serialize, deserialize +from django.contrib.auth import get_user_model +from django.conf import settings +from apps.iiif.annotations.tests.factories import AnnotationFactory +from apps.iiif.canvases.tests.factories import CanvasFactory +from apps.iiif.manifests.tests.factories import ManifestFactory +from apps.iiif.annotations.models import Annotation +from apps.readux.tests.factories import UserAnnotationFactory +from apps.users.tests.factories import UserFactory + +User = get_user_model() + + +class AnnotationSerializerTests(TestCase): + def setUp(self): + """_summary_""" + self.annotation = Annotation() + + def test_user_annotation_svg_comment_fragment_deserialization(self): + user = UserFactory.create() + canvas = CanvasFactory.create(manifest=ManifestFactory.create()) + user_annotation = UserAnnotationFactory.create( + owner=user, + canvas=canvas, + svg='