diff --git a/.env.dist b/.env.dist new file mode 100644 index 000000000..e4f93314d --- /dev/null +++ b/.env.dist @@ -0,0 +1,5 @@ +# Use the Official DBpedia Spotlight API ... +DBPEDIA_SPOTLIGHT_URL="https://api.dbpedia-spotlight.org/en/annotate" + +# ... or a custom DBpedia Spotlight API endpoint +#DBPEDIA_SPOTLIGHT_URL="https://example.org/rest/annotate" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b4cd98587..db40dc767 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,19 +35,19 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 with: platforms: ${{ inputs.platforms }} - name: Login to container registry if: ${{ inputs.push }} - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} @@ -66,7 +66,7 @@ jobs: - name: Get tagging metadata id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: images: ${{ steps.image_name.outputs.full }} flavor: @@ -81,7 +81,7 @@ jobs: org.opencontainers.image.documentation="${{ github.server_url }}/{{ github.repository }}/blob/main/${{ matrix.service }}/README.md" - name: Build and push container image - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: platforms: ${{ inputs.platforms }} provenance: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0c36c884c..5bc9576a5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Check for stable semver id: semver diff --git a/.gitignore b/.gitignore index 0b15f9da5..a993fea20 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ pids *.pid *.seed *.pid.lock -public/uploads +/webserver/public/uploads # Directory for instrumented libs generated by jscoverage/JSCover lib-cov @@ -266,3 +266,9 @@ ser_data.txt coursemapper-kg/app/algorithms/elmo_2x4096_512_2048cnn_2xhighway_options.json coursemapper-kg/app/algorithms/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5 +coursemapper-kg/=2.2.2 +venv* +.DS_Store + +coursemapper-kg/app +webserver/public/uploads/pdfs diff --git a/.k8s/base/coursemapper-kg/concept-map/deployment.yaml b/.k8s/base/coursemapper-kg/concept-map/deployment.yaml index 9f20eec2c..4c82b0099 100644 --- a/.k8s/base/coursemapper-kg/concept-map/deployment.yaml +++ b/.k8s/base/coursemapper-kg/concept-map/deployment.yaml @@ -32,6 +32,9 @@ spec: value: postgres://postgres:password@$(WP_PG_SERVICE):5432 - name: WIKIPEDIA_FALLBACK value: "false" + envFrom: + - configMapRef: + name: dbpedia-spotlight resources: requests: cpu: 100m diff --git a/.k8s/base/coursemapper-kg/concept-map/kustomization.yaml b/.k8s/base/coursemapper-kg/concept-map/kustomization.yaml index 9cc4dd8f1..ce0569e76 100644 --- a/.k8s/base/coursemapper-kg/concept-map/kustomization.yaml +++ b/.k8s/base/coursemapper-kg/concept-map/kustomization.yaml @@ -1,8 +1,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -commonLabels: - app.kubernetes.io/component: coursemapper-kg-concept-map +labels: +- includeSelectors: true + pairs: + app.kubernetes.io/component: coursemapper-kg-concept-map resources: - deployment.yaml diff --git a/.k8s/base/coursemapper-kg/pgadmin/ingress.yaml b/.k8s/base/coursemapper-kg/pgadmin/ingress.yaml index a7a9b17f5..b9194261e 100644 --- a/.k8s/base/coursemapper-kg/pgadmin/ingress.yaml +++ b/.k8s/base/coursemapper-kg/pgadmin/ingress.yaml @@ -5,7 +5,6 @@ metadata: annotations: kubernetes.io/tls-acme: "true" cert-manager.io/cluster-issuer: letsencrypt-prod - traefik.ingress.kubernetes.io/router.middlewares: default-headers@kubernetescrd spec: rules: - host: pgadmin.$(FQDN) diff --git a/.k8s/base/coursemapper-kg/pgadmin/kustomization.yaml b/.k8s/base/coursemapper-kg/pgadmin/kustomization.yaml index 06b674bf3..ab2cde095 100644 --- a/.k8s/base/coursemapper-kg/pgadmin/kustomization.yaml +++ b/.k8s/base/coursemapper-kg/pgadmin/kustomization.yaml @@ -1,8 +1,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -commonLabels: - app.kubernetes.io/component: coursemapper-kg-pgadmin +labels: +- includeSelectors: true + pairs: + app.kubernetes.io/component: coursemapper-kg-pgadmin resources: - deployment.yaml diff --git a/.k8s/base/coursemapper-kg/recommendation/kustomization.yaml b/.k8s/base/coursemapper-kg/recommendation/kustomization.yaml index 8486cfc25..333bea8aa 100644 --- a/.k8s/base/coursemapper-kg/recommendation/kustomization.yaml +++ b/.k8s/base/coursemapper-kg/recommendation/kustomization.yaml @@ -1,8 +1,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -commonLabels: - app.kubernetes.io/component: coursemapper-kg-recommendation +labels: +- includeSelectors: true + pairs: + app.kubernetes.io/component: coursemapper-kg-recommendation resources: - deployment.yaml diff --git a/.k8s/base/coursemapper-kg/wp-pg/kustomization.yaml b/.k8s/base/coursemapper-kg/wp-pg/kustomization.yaml index 9b63f6ec1..f833a4b5e 100644 --- a/.k8s/base/coursemapper-kg/wp-pg/kustomization.yaml +++ b/.k8s/base/coursemapper-kg/wp-pg/kustomization.yaml @@ -1,8 +1,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -commonLabels: - app.kubernetes.io/component: coursemapper-kg-wp-pg +labels: +- includeSelectors: true + pairs: + app.kubernetes.io/component: coursemapper-kg-wp-pg resources: - deployment.yaml diff --git a/.k8s/base/ingress.yaml b/.k8s/base/ingress.yaml index e6db0595f..233df0654 100644 --- a/.k8s/base/ingress.yaml +++ b/.k8s/base/ingress.yaml @@ -6,7 +6,6 @@ metadata: description: CourseMapper-webserver kubernetes.io/tls-acme: "true" cert-manager.io/cluster-issuer: letsencrypt-prod - traefik.ingress.kubernetes.io/router.middlewares: default-headers@kubernetescrd spec: rules: - host: $(FQDN) diff --git a/.k8s/base/kustomization.yaml b/.k8s/base/kustomization.yaml index 26380b1dc..277483d8c 100644 --- a/.k8s/base/kustomization.yaml +++ b/.k8s/base/kustomization.yaml @@ -19,6 +19,9 @@ configMapGenerator: - name: ingress literals: - host=example.org +- name: dbpedia-spotlight + literals: [] + # - DBPEDIA_SPOTLIGHT_URL=https://example.org/rest/annotate images: - name: coursemapper-kg-concept-map diff --git a/.k8s/base/mongodb/deployment.yaml b/.k8s/base/mongodb/deployment.yaml index 2f3bdc6bf..329cf4ab1 100644 --- a/.k8s/base/mongodb/deployment.yaml +++ b/.k8s/base/mongodb/deployment.yaml @@ -26,10 +26,10 @@ spec: resources: requests: memory: 128Mi - cpu: 10m + cpu: 500m limits: memory: 512Mi - cpu: 2 + cpu: 1150m ephemeral-storage: 1Gi containers: - name: mongodb diff --git a/.k8s/base/webserver/neo4j/deployment.yaml b/.k8s/base/webserver/neo4j/deployment.yaml index e41241c85..ba407f70c 100644 --- a/.k8s/base/webserver/neo4j/deployment.yaml +++ b/.k8s/base/webserver/neo4j/deployment.yaml @@ -20,10 +20,10 @@ spec: resources: requests: cpu: 12m - memory: 800Mi + memory: 1100Mi limits: cpu: 4 - memory: 1.2Gi + memory: 1.5Gi volumeMounts: - name: data mountPath: "/data" diff --git a/.k8s/base/webserver/neo4j/kustomization.yaml b/.k8s/base/webserver/neo4j/kustomization.yaml index 6c7de26fe..e43fdaa8d 100644 --- a/.k8s/base/webserver/neo4j/kustomization.yaml +++ b/.k8s/base/webserver/neo4j/kustomization.yaml @@ -1,8 +1,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -commonLabels: - app.kubernetes.io/component: webserver-neo4j +labels: +- includeSelectors: true + pairs: + app.kubernetes.io/component: webserver-neo4j resources: - deployment.yaml diff --git a/.k8s/base/webserver/redis/kustomization.yaml b/.k8s/base/webserver/redis/kustomization.yaml index ace205743..3d8d280d7 100644 --- a/.k8s/base/webserver/redis/kustomization.yaml +++ b/.k8s/base/webserver/redis/kustomization.yaml @@ -1,8 +1,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -commonLabels: - app.kubernetes.io/component: webserver-redis +labels: +- includeSelectors: true + pairs: + app.kubernetes.io/component: webserver-redis resources: - deployment.yaml diff --git a/.k8s/base/webserver/web/deployment.yaml b/.k8s/base/webserver/web/deployment.yaml index 6389d4181..6dbc55f1f 100644 --- a/.k8s/base/webserver/web/deployment.yaml +++ b/.k8s/base/webserver/web/deployment.yaml @@ -28,6 +28,9 @@ spec: httpGet: path: /api/healthz port: 8080 + timeoutSeconds: 30 + failureThreshold: 5 + periodSeconds: 60 env: - name: PORT value: "8080" @@ -39,6 +42,13 @@ spec: value: $(REDIS_SERVICE) - name: REDIS_PORT value: "6379" + - name: MEMORY_REQUEST + valueFrom: + resourceFieldRef: + divisor: "1Mi" + resource: requests.memory + - name: NODE_OPTIONS + value: "--max-old-space-size=$(MEMORY_REQUEST)" envFrom: - secretRef: name: webserver @@ -54,11 +64,11 @@ spec: subPath: videos resources: requests: - memory: 360Mi + memory: 2Gi cpu: 100m limits: - memory: 1Gi - cpu: "1" + memory: 2.5Gi + cpu: "1.15" ephemeral-storage: 1Gi volumes: - name: webserver-uploads diff --git a/.k8s/base/webserver/web/kustomization.yaml b/.k8s/base/webserver/web/kustomization.yaml index 9c06e0994..a1eb4f48d 100644 --- a/.k8s/base/webserver/web/kustomization.yaml +++ b/.k8s/base/webserver/web/kustomization.yaml @@ -1,8 +1,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -commonLabels: - app.kubernetes.io/component: webserver-web +labels: +- includeSelectors: true + pairs: + app.kubernetes.io/component: webserver-web resources: - deployment.yaml diff --git a/Makefile b/Makefile index 2ca4f25da..1af37ab62 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ help: @cat $(MAKEFILE_LIST) | docker run --rm -i --platform linux/amd64 xanders/make-help +export COMPOSE_BAKE=1 compose = docker compose -f compose.yaml all: clean build run @@ -32,7 +33,7 @@ build: # Push container images to remote registry push: - @docker compose push + @$(compose) push # Start all services for development using Tilt for live container updates tilt: diff --git a/README.md b/README.md index bec01fd8d..1c248c35b 100644 --- a/README.md +++ b/README.md @@ -6,27 +6,27 @@ CourseMapper is a collaborative course annotation and analytics platform that fo ## 🚀 Get Started -#### Live instances +#### 🌎 Live instances - Production: [coursemapper.de](https://coursemapper.de/) ([branch `main`](https://github.com/ude-soco/CourseMapper-webserver/tree/main)) - ![status](https://argocd.cluster.soco.inko.cloud/api/badge?name=cmw-prod) + ![production status](https://argocd.soco.ude.systems/api/badge?name=cmw-prod) - Preview: [br-dev.coursemapper.de](https://br-dev.coursemapper.de/) ([branch `dev`](https://github.com/ude-soco/CourseMapper-webserver/tree/dev)) - ![status](https://argocd.cluster.soco.inko.cloud/api/badge?name=cmw-br-dev) + ![branch preview status](https://argocd.soco.ude.systems/api/badge?name=cmw-br-dev) *Note:* Stable [releases](https://github.com/ude-soco/CourseMapper-webserver/releases) are currently not running in production. -#### Build and run +#### 🐳 Compose -- `make up` to run the application using _Docker Compose_ -- `make tilt` to automatically rebuild during development using _Tilt_ -- `make mounted` to run processes using _Docker Compose_, but mount source code from host machine -- see the manual below to install dependencies and run processes _locally, without containers_ - -Visit the [proxy service on port 8000](http://localhost:8000/) to use the application. +1. Set up `.env` file (see `.env.dist`) +2. Build and run + - `make up` to run the application using _Docker Compose_ + - `make tilt` to automatically rebuild during development using _Tilt_ + - `make mounted` to run processes using _Docker Compose_, but mount source code from host machine +3. Visit the [proxy service on port 8000](http://localhost:8000/) to use the application. ## 🖥️ Application stack diff --git a/coursemapper-kg/concept-map/Dockerfile b/coursemapper-kg/concept-map/Dockerfile index dd9ba2d09..f2f77631d 100644 --- a/coursemapper-kg/concept-map/Dockerfile +++ b/coursemapper-kg/concept-map/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.6 +# syntax=docker/dockerfile:1.15 FROM alpine as MODELS WORKDIR /download @@ -7,7 +7,7 @@ ADD https://uni-duisburg-essen.sciebo.de/s/nO06q2wY0t5h8SO/download stanford-cor RUN unzip stanford-corenlp-full-2018-02-27.zip -d stanford-corenlp-full-2018-02-27 && rm stanford-corenlp-full-2018-02-27.zip -from python:3.11.5 +FROM python:3.11.5 # APT requires archive URLs COPY debian/sources.list /etc/apt/sources.list @@ -30,10 +30,10 @@ USER 1000 WORKDIR /home/app # Configure Python environment -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 -ENV PIPENV_VENV_IN_PROJECT 1 -ENV PIPENV_VERBOSITY -1 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV PIPENV_VENV_IN_PROJECT=1 +ENV PIPENV_VERBOSITY=-1 ENV VIRTUAL_ENV=/home/app/.venv RUN python -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" @@ -42,7 +42,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" COPY --chown=app:app Pipfile* ./ RUN --mount=type=cache,sharing=private,target=/home/app/.cache/pip,uid=1000 \ --mount=type=cache,sharing=private,target=/home/app/.cache/pypoetry,uid=1000 < /etc/apt/apt.conf.d/keep-cache + apt-get update -yq && + apt-get install -y --no-install-recommends $BUILD_DEPS && + + # Add PostgreSQL package source (runs `apt-get update`) + /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y && + + apt-get install -y --no-install-recommends $RUNTIME_DEPS && + rm -rf /var/lib/apt/lists/* +EOF -# Install the required packages -RUN apt-get update && apt install -y postgresql-common -RUN /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y -RUN apt-get update && apt-get install -y --no-install-recommends \ - lbzip2 \ - postgresql-client-16 \ - rclone \ - && rm -rf /var/lib/apt/lists/* -RUN pip install --no-color pipenv==2023.12.1 +# Install Python packages +COPY Pipfile Pipfile.lock ./ +RUN pip install --no-color --disable-pip-version-check --no-cache-dir pipenv==2023.12.1 RUN pipenv install --deploy +# Remove build dependencies +RUN apt-get remove -yq $BUILD_DEPS + # Copy config files COPY config/rclone.conf /root/.config/rclone/rclone.conf diff --git a/coursemapper-kg/recommendation/Dockerfile b/coursemapper-kg/recommendation/Dockerfile index ff8dda370..81554057a 100644 --- a/coursemapper-kg/recommendation/Dockerfile +++ b/coursemapper-kg/recommendation/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.6 +# syntax=docker/dockerfile:1.15 FROM python:3.11.5 # APT requires archive URLs @@ -34,7 +34,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" COPY --chown=app:app Pipfile* . RUN --mount=type=cache,sharing=private,target=/home/app/.cache/pip,uid=1000 \ --mount=type=cache,sharing=private,target=/home/app/.cache/pipenv,uid=1000 < adj) - adj.multiply(adj.T > adj) adj = self.normalize(adj) + sp.eye(adj.shape[0]) - logger.info(adj.A) + # logger.info(adj.A) # GCN Multiply Adjacency matrix and initial embedding matrix # mutiply Adjacency matrix and initial embedding matrix, output is new embedding matrix # The new embedding of a node is obtained by aggregating the embeddings of its first hop neighbours @@ -79,7 +80,7 @@ def load_data(self): # The final embedding of a node is obtained by aggregating the embeddings of its first and second hop neighbours output = np.dot(adj, output) final_embeddings = output.A - logger.info(final_embeddings) + # logger.info(final_embeddings) # Extract original ids of nodes idx = np.array(idx_features[:, 1], dtype=np.dtype(str)) @@ -122,8 +123,8 @@ def load_data(self): def normalize(self, mx): rowsum = np.array(mx.sum(1)) - d_inv = np.power(rowsum, -0.5).flatten() - d_inv[np.isinf(d_inv)] = 0.0 + d_inv = np.power(rowsum, -0.5, where=rowsum != 0).flatten() + d_inv[rowsum.flatten() == 0] = 0.0 d_mat_inv = sp.diags(d_inv) norm_adj = d_mat_inv.dot(mx) norm_adj = norm_adj.dot(d_mat_inv) diff --git a/coursemapper-kg/recommendation/app/services/course_materials/db/neo4_db.py b/coursemapper-kg/recommendation/app/services/course_materials/db/neo4_db.py index d058cff19..295c90cdd 100644 --- a/coursemapper-kg/recommendation/app/services/course_materials/db/neo4_db.py +++ b/coursemapper-kg/recommendation/app/services/course_materials/db/neo4_db.py @@ -3,6 +3,7 @@ import re import logging from log import LOG +import json #set pythonHashSeed to zero to have same hashed value if same input has been given import os @@ -12,6 +13,8 @@ # os.environ['PYTHONHASHSEED'] = '0' # os.execv(sys.executable, [sys.executable] + sys.argv) +from datetime import datetime + logger = LOG(name=__name__, level=logging.DEBUG) @@ -66,12 +69,15 @@ def create_video_resource(tx, node, recommendation_type=''): """ logger.info( "Creating youtube resource '%s'" % node["id"]) + tx.run( """MERGE (c:Resource:Video {rid: $rid, uri: $uri, title: $title, description: $description, description_full: $description_full, keyphrases: $keyphrases, text: $text, document_embedding: $document_embedding, keyphrase_embedding: $keyphrase_embedding, similarity_score: $similarity_score, thumbnail: $thumbnail, duration: $duration, views: $views, publish_time: $pub_time, helpful_count: $helpful_count, - not_helpful_count: $not_helpful_count})""", + not_helpful_count: $not_helpful_count, saves_count: $saves_count, like_count: $like_count, channel_title: $channel_title + }) + """, rid=node["id"], uri="https://www.youtube.com/embed/%s?autoplay=1" % node["id"], title=node["title"], @@ -87,7 +93,11 @@ def create_video_resource(tx, node, recommendation_type=''): keyphrase_embedding=str(node["keyphrase_embedding"] if "keyphrase_embedding" in node.index else ""), document_embedding=str(node["document_embedding"] if "document_embedding" in node.index else ""), helpful_count=0, - not_helpful_count=0) + not_helpful_count=0, + saves_count=0, + like_count=node["like_count"], + channel_title=node["channel_title"] + ) def create_wikipedia_resource(tx, node, recommendation_type=''): @@ -99,7 +109,9 @@ def create_wikipedia_resource(tx, node, recommendation_type=''): """MERGE (c:Resource:Article {rid: $rid, uri: $uri, title: $title, abstract:$abstract, keyphrases: $keyphrases, text: $text, document_embedding: $document_embedding, keyphrase_embedding: $keyphrase_embedding, similarity_score: $similarity_score, helpful_count: $helpful_count, - not_helpful_count: $not_helpful_count})""", + not_helpful_count: $not_helpful_count, saves_count: $saves_count + }) + """, rid=node["id"], uri=node["id"], title=node["title"], @@ -110,8 +122,30 @@ def create_wikipedia_resource(tx, node, recommendation_type=''): keyphrase_embedding=str(node["keyphrase_embedding"] if "keyphrase_embedding" in node.index else ""), document_embedding=str(node["document_embedding"] if "document_embedding" in node.index else ""), helpful_count=0, - not_helpful_count=0) + not_helpful_count=0, + saves_count=0 + ) +def create_external_source_resource(tx, node): + """ + Create ExternalSource Node + """ + logger.info( + "Creating ExternalSource resource '%s'" % node["id"]) + tx.run( + """MERGE (c:Resource:ExternalSource {rid: $rid, uri: $uri, + publish_time: $created_at, cid: $cid, description: $description, helpful_count: $helpful_count, + not_helpful_count: $not_helpful_count, saves_count: $saves_count})""", + rid=node["uri"], + uri=node["uri"], + publish_time=node["created_at"], + description=node["description"], + cid=node["cid"], + helpful_count=0, + not_helpful_count=0, + saves_count=0 + ) + def create_concept_relationships(tx, node): """ @@ -197,12 +231,16 @@ def edit_resource(tx, resource, recommendation_type): """ logger.info( "Editing resource '%s'" % resource["id"]) + tx.run(""" MATCH (b: Resource) WHERE b.rid = $rid - SET b.similarity_score = $similarity_score + SET b.similarity_score = $similarity_score, b.views = $views, b.like_count = $like_count, b.channel_title = $channel_title RETURN b """, rid=resource["id"], + views=resource["views"], + like_count=resource["like_count"], + channel_title=resource["channel_title"], concepts=resource["keyphrases"] if "keyphrases" in resource.index else [], similarity_score=resource[recommendation_type] if recommendation_type in resource.index else 0, keyphrase_embedding=str(resource["keyphrase_embedding"] if "keyphrase_embedding" in resource.index else ""), @@ -339,11 +377,23 @@ def get_user(tx, uid): "Get user '%s'" % uid) result = tx.run( "MATCH (u:User) WHERE u.uid = $uid RETURN u", - uid=uid).data() + uid=uid).single() return list(result) +# def get_user_v2(tx, uid): +# """ +# """ +# logger.info( +# "Get user '%s'" % uid) +# result = tx.run( +# "MATCH (u:User) WHERE u.uid = $uid RETURN u.uid as uid", +# uid=uid).single() + +# return result + + def retrieve_all_concepts(tx, mid): """ """ @@ -440,6 +490,18 @@ def create_user(tx, user_id): type="user", embedding="") +def create_user_v2(tx, user): + """ + """ + user_id = tx.run( + """MERGE (u:User {name: $name, uid: $uid, type: $type, email: $userEmail, embedding:$embedding}) RETURN u.uid""", + name=user["name"], + uid=user["id"], + type="user", + userEmail=user["user_email"], + embedding="").single() + return user_id + def retrieve_user_concept_relationships(tx, uid, relation_type): """ @@ -704,6 +766,24 @@ def get_or_create_user(self, user_id, username="", user_email=""): session.close() self.close() + # def get_or_create_user_v2(self, user): + # """ + # """ + # session = self.driver.session() + # tx = session.begin_transaction() + # user = None + + # # _uid = get_user(tx, user_id) + # user = get_user_v2(tx, user["id"]) + # print("get_user_v2 -> ->", user) + + # if user == None: + # user = create_user_v2(tx, user) + # # create_user(tx, user) + # print("user -> ->", user) + + # return user + def get_concepts_and_relationships(self, materialId): """ """ @@ -1524,7 +1604,7 @@ def get_top_n_dnu_concepts(self, user_id, material_id, top_n=5): mid=material_id).data() # concepts = list(result) logger.info("User dnu concepts") - logger.info(concepts) + # logger.info(concepts) concepts.sort(key=lambda x: x["weight"], reverse=True) @@ -1704,6 +1784,948 @@ def update_concept(self, node): cid=node["id"], mid=node["mid"], rank=node["rank"]) + session.commit() session.close() + + ########### + # boby024 # + ########### + + def get_or_create_user_v2(self, user): + """ + """ + tx = self.driver.session() + user_node = tx.run( + "MATCH (u:User) WHERE u.uid = $uid RETURN u.uid as uid", + uid=user["user_id"] + ).single() + + if user_node is None: + user_node = tx.run( + """MERGE (u:User {name: $name, uid: $uid, type: $type, email: $userEmail, embedding:$embedding}) RETURN u.uid""", + name=user["name"], + uid=user["user_id"], + type="user", + userEmail=user["user_email"], + embedding="" + ).single() + + return user_node + + def create_or_update_video_resource(self, tx, node: dict, + recommendation_type='', + update_embedding_values=False, + update_detail_found=False + ): + ''' + Creating Resource YouTube + r.similarity_score = $similarity_score, + ''' + # logger.info(" Creating Resource YouTube") + try: + """ + if update_detail_found == True: + tx.run( + ''' + MATCH (r:Resource: Video) + WHERE r.rid = $rid + SET r.title = $title, r.description = $description, r.description_full = $description_full, + r.text = $text, r.duration = $duration, r.views = $views, like_count = $like_count, + r.channel_title = $channel_title, r.updated_at = $updated_at, + r.keyphrases = $keyphrases, r.keyphrase_embedding = $keyphrase_embedding, + r.document_embedding = $document_embedding + ''', + rid=node["rid"], + title=node["title"], + description=node["description"], + description_full=node["description_full"], + text=node["text"], + duration=node["duration"], + views=node["views"], + like_count=node["like_count"], + channel_title=node["channel_title"], + updated_at=datetime.now().isoformat(), + keyphrases=[], + keyphrase_embedding="", + document_embedding="" + ) + """ + + if update_embedding_values == True: # node.get("keyphrases") != None or node.get("document_embedding") != None or node.get("keyphrase_embedding") != None: + tx.run( + ''' + MATCH (r:Resource: Video) + WHERE r.rid = $rid + SET r.keyphrases = $keyphrases, r.keyphrase_embedding = $keyphrase_embedding, r.document_embedding = $document_embedding, + r.keyphrases_infos = $keyphrases_infos + + ''', + rid=node["rid"], + keyphrases=node["keyphrases"] if "keyphrases" in node else [], + keyphrase_embedding=node["keyphrase_embedding"] if "keyphrase_embedding" in node else [], + document_embedding=node["document_embedding"] if "document_embedding" in node else [], + keyphrases_infos=node["keyphrases_infos"] if "keyphrases_infos" in node else "" + ) + else: + tx.run( + ''' + MERGE (r:Resource:Video {rid: $rid}) + ON CREATE SET + r.uri = $uri, r.title = $title, r.description = $description, r.description_full = $description_full, r.text = $text, + r.keyphrases = $keyphrases, r.document_embedding = $document_embedding, r.keyphrase_embedding = $keyphrase_embedding, + r.thumbnail = $thumbnail, r.duration = $duration, r.views = $views, + r.publish_time = $pub_time, r.channel_title = $channel_title, r.like_count = $like_count, + r.helpful_count = $helpful_count, r.not_helpful_count = $not_helpful_count, r.saves_count = $saves_count, + r.updated_at = $updated_at + ON MATCH SET + r.title = $title, r.description = $description, r.description_full = $description_full, r.text = $text, + r.keyphrases = $keyphrases, r.document_embedding = $document_embedding, r.keyphrase_embedding = $keyphrase_embedding, + r.thumbnail = $thumbnail, r.duration = $duration, r.views = $views, + r.publish_time = $pub_time, r.channel_title = $channel_title, r.like_count = $like_count, + r.updated_at = $updated_at + ''', + rid=node["id"], + uri="https://www.youtube.com/embed/%s?autoplay=1" % node["id"], + title=node["title"], + description=node["description"], + description_full=node["description_full"], + thumbnail="https://i.ytimg.com/vi/%s/hqdefault.jpg" % node["id"], + text=node["text"], + duration=node["duration"], + views=node["views"], + pub_time=node["publishTime"], + # similarity_score=node[recommendation_type] if recommendation_type in node.index else 0, + keyphrases=node["keyphrases"] if "keyphrases" in node else [], + keyphrase_embedding=node["keyphrase_embedding"] if "keyphrase_embedding" in node else [], + document_embedding=node["document_embedding"] if "document_embedding" in node else [], + helpful_count=node["helpful_count"] if "helpful_count" in node else 0, + not_helpful_count=node["not_helpful_count"] if "not_helpful_count" in node else 0, + saves_count=node["saves_count"] if "saves_count" in node else 0, + like_count=node["like_count"], + channel_title=node["channel_title"], + updated_at=datetime.now().isoformat(), + keyphrases_infos=node["keyphrases_infos"] if "keyphrases_infos" in node else "" + ) + except Exception as e: + print(e) + pass + + def create_or_update_wikipedia_resource(self, tx, node, recommendation_type='', + update_embedding_values=False, + update_detail_found=False + ): + ''' + Creating Resource Wikipedia + ''' + # logger.info("Creating Resource Wikipedia") + try: + """ + if update_detail_found == True: + tx.run( + ''' + MATCH (r:Resource: Article) + WHERE r.rid = $rid + SET r.title = $title, r.abstract = $abstract, r.text = $text, updated_at = $updated_at, + r.keyphrases = $keyphrases, r.keyphrase_embedding = $keyphrase_embedding, + r.document_embedding = $document_embedding + ''', + rid=node["rid"], + title=node["title"], + abstract=node["abstract"], + text=node["text"], + updated_at=datetime.now().isoformat(), + keyphrases=node["keyphrases"] if "keyphrases" in node else [], + keyphrase_embedding=str(node["keyphrase_embedding"] if "keyphrase_embedding" in node else ""), + document_embedding=str(node["document_embedding"] if "document_embedding" in node else ""), + ) + """ + + if update_embedding_values == True: # node.get("keyphrases") != None or node.get("document_embedding") != None or node.get("keyphrase_embedding") != None: + tx.run( + ''' + MATCH (r:Resource: Article) + WHERE r.rid = $rid + SET r.keyphrases = $keyphrases, r.keyphrase_embedding = $keyphrase_embedding, r.document_embedding = $document_embedding, + r.keyphrases_infos = $keyphrases_infos + ''', + rid=node["rid"], + keyphrases=node["keyphrases"] if "keyphrases" in node else [], + keyphrase_embedding=node["keyphrase_embedding"] if "keyphrase_embedding" in node else [], + document_embedding=node["document_embedding"] if "document_embedding" in node else [], + keyphrases_infos=node["keyphrases_infos"] if "keyphrases_infos" in node else "" + ) + else: + tx.run( + ''' + MERGE (r:Resource:Article {rid: $rid}) + ON CREATE SET + r.uri = $uri, r.title = $title, r.abstract = $abstract, r.text = $text, + r.keyphrases = $keyphrases, r.document_embedding = $document_embedding, r.keyphrase_embedding = $keyphrase_embedding, + r.helpful_count = $helpful_count, r.not_helpful_count = $not_helpful_count, r.saves_count = $saves_count, + r.updated_at = $updated_at + ON MATCH SET + r.uri = $uri, r.title = $title, r.abstract = $abstract, r.text = $text, + r.keyphrases = $keyphrases, r.document_embedding = $document_embedding, r.keyphrase_embedding = $keyphrase_embedding, + r.updated_at = $updated_at + ''', + rid=node["id"], + uri=node["id"], + title=node["title"], + abstract=node["abstract"], + keyphrases=node["keyphrases"] if "keyphrases" in node else [], + text=node["text"], + # similarity_score=node[recommendation_type] if recommendation_type in node.index else 0, + keyphrase_embedding=node["keyphrase_embedding"] if "keyphrase_embedding" in node else [], + document_embedding=node["document_embedding"] if "document_embedding" in node else [], + helpful_count=node["helpful_count"] if "helpful_count" in node else 0, + not_helpful_count=node["not_helpful_count"] if "not_helpful_count" in node else 0, + saves_count=node["saves_count"] if "saves_count" in node else 0, + updated_at=datetime.now().isoformat(), + keyphrases_infos=node["keyphrases_infos"] if "keyphrases_infos" in node else "" + ) + except Exception as e: + print(e) + pass + + def get_top_n_concept_by_slide_id(self, slide_id: str, names: list = None, top_n=5): + ''' + Get top n concept by slide ID and concept names + ''' + concepts = [] + if names: + with self.driver.session() as session: + logger.info("Get top n concept by slide ID") + concepts = session.run( + ''' + MATCH p=(s: Slide)-[r: CONTAINS]->(c: Concept) + WHERE s.sid = $slide_id AND c.name IN $names + RETURN ID(c) as id, c.cid as cid, c.name as name, c.weight as weight + ORDER BY c.weight DESC + LIMIT $top_n + ''', + slide_id=slide_id, + names=names, + top_n=top_n + ).data() + else: + with self.driver.session() as session: + logger.info("Get top n concept by slide ID and concept names") + concepts = session.run( + ''' + MATCH p=(s: Slide)-[r: CONTAINS]->(c: Concept) + WHERE s.sid = $slide_id + RETURN ID(c) as id, c.cid as cid, c.name as name, c.weight as weight + ORDER BY c.weight DESC + ''', + slide_id=slide_id + ).data() + + return concepts + + def create_concept_modified(self, cid: str): + ''' + Creating node 'Concept_modified' + ''' + tx = self.driver.session() + concept = None + + # get original Concept node + concept_original = tx.run( + ''' + MATCH (c:Concept) WHERE c.cid = $cid + RETURN c.cid as cid, c.final_embedding as final_embedding + ''', + cid=cid + ).single() + + if concept_original: + concept = tx.run( + ''' + MERGE (c:Concept_modified { cid:$cid, final_embedding:$final_embedding }) + RETURN ID(c) as node_id, c.cid as cid + ''', + cid=concept_original["cid"], + final_embedding=concept_original["final_embedding"] + ).single() + concept = {"node_id": concept["node_id"], "cid": concept["cid"]} + + return concept + + def update_rs_btw_user_and_cm(self, user_id: str, cid: str, weight: float, mid: str, status: str, only_status=False): + ''' + Create or Update relationship between nodes 'User' and 'Concept_modified' + typ: remove (from understood_list) + ''' + tx = self.driver.session() + + if only_status == True: + tx.run( + ''' + MATCH (a:User)-[r:HAS_MODIFIED]->(b:Concept_modified) + WHERE a.uid = $user_id AND r.user_id = $user_id AND b.cid = $cid + SET r.status = $status + ''', + user_id=user_id, + cid=cid, + status=status + ).single() + + else: + r_detail = tx.run( + ''' + MATCH (a:User {uid: $user_id}), (b:Concept_modified {cid: $cid}) + MERGE (a)-[r:HAS_MODIFIED]->(b) + ON CREATE SET r.user_id = $user_id, r.weight = $weight, r.mid = $mid, r.status = $status + ON MATCH SET r.user_id = $user_id, r.weight = $weight, r.mid = $mid, r.status = $status + RETURN ID(b) as cm_id, b.cid as cid, r.weight as weight, r.mid as mid, r.status as status + ''', + user_id=user_id, + cid=cid, + weight=weight, + mid=mid, + status=status + ).single() + + r_detail = { "cm_id": r_detail["cm_id"], "cid": r_detail["cid"], + "weight": r_detail["weight"], "mid": r_detail["mid"], + "status": r_detail["status"] + } + return r_detail + + def update_rs_btw_user_and_cms(self, user_id: str, cids: list, special_status): + ''' + Create or Update relationship between nodes 'User' and list of cids ('Concept_modified') given + relationship: dnu_reset + for dnus that should be used for recommendations + ''' + logger.info("User Concepts not used: DNU Reset") + with self.driver.session() as session: + result = session.run( + ''' + MATCH (a:User)-[r:HAS_MODIFIED]->(b:Concept_modified) + WHERE r.user_id = $user_id AND NOT b.cid IN $cids + SET r.status = "dnu_reset" + ''', + user_id=user_id, + cids=cids + ) + + def get_user_embedding_with_concept_modified(self, user_id: str, mid: str, status: str): + """ + Update User Embedding value based on nodes 'Concept_modified' + """ + embedding = "" + tx = self.driver.session() + + # Find concept embeddings that user doesn't understand + embeddings = tx.run( + ''' + MATCH (a:User)-[r:HAS_MODIFIED]->(b:Concept_modified) + WHERE a.uid = $user_id AND r.user_id = $user_id AND + r.mid = $mid AND r.status = $status + RETURN b.final_embedding as embedding, r.weight as weight + ''', + user_id=user_id, + mid=mid, + status=status + ).data() + + # If the user does not have concepts that he does not understand, the list is empty + if len(embeddings) == 0: + tx.run( + ''' + MATCH (u:User) WHERE u.uid=$user_id set u.embedding_resources=$embedding + ''', + user_id=user_id, + embedding="" + ) + # logger.info("reset user embedding") + else: + sum_embeddings = 0 + sum_weights = 0 + # Convert string type to array type 'np.array' + # Sum and average these concept embeddings to get user embedding + for embedding in embeddings: + list1 = embedding["embedding"].split(',') + list2 = [] + for j in list1: + list2.append(float(j)) + arr = np.array(list2) + sum_embeddings = sum_embeddings + arr * embedding["weight"] + sum_weights = sum_weights + embedding["weight"] + # The weighted average of final embeddings of all dnu concepts + average = np.divide(sum_embeddings, sum_weights) + embedding=','.join(str(i) for i in average) + + tx.run("""MATCH (u:User) WHERE u.uid=$user_id set u.embedding_resources=$embedding""", + user_id=user_id, + embedding=embedding + ) + return embedding + + def user_rates_resources(self, rating: dict): + ''' + User rates Resource(s) + rating: { + "user_id": " 43dukl8", + "cid": "dsfis23sd" + "value": "HELPFUL" | "NOT_HELPFUL", + "rid": "bnm565j", + "mid": "sdddfe", + "reset": True # to undo any rating + } + ''' + logger.info("Saving or Removing: User Resource") + tx = self.driver.session() + + reset_status = False + if rating.get("reset") != None and rating["reset"] == True: + tx.run( + # ''' + # WITH $cids AS elementsToRemove + # MATCH (a:User)-[r:HAS_RATED]->(b:Resource) + # WHERE r.user_id = $user_id AND r.rid = $rid AND r.value = $value + # SET r.cids = [e IN r.elements WHERE NOT e IN elementsToRemove] + # ''', + ''' + MATCH (a:User)-[r:HAS_RATED]->(b:Resource) + WHERE r.user_id = $user_id AND r.rid = $rid AND r.value = $value + DELETE r + ''', + user_id=rating["user_id"], + rid=rating["rid"], + value=rating["value"] + ) + reset_status = True + + # Add or Update rating + elif rating["value"] != "HELPFUL": + tx.run( + ''' + MATCH (a:User {uid: $user_id}), (b:Resource {rid: $rid}) + MERGE (a)-[r:HAS_RATED {user_id: $user_id, rid: $rid}]->(b) + ON CREATE SET r.cids = $cids, r.value = $value + ON MATCH SET r.user_id = $user_id, r.rid = $rid, r.value = $value, r.cids = $cids + ''', + user_id=rating["user_id"], + rid=rating["rid"], + value=rating["value"], + cids=[] + ) + + else: + # apoc.coll.toSet(apoc.coll.union(r.cids, $cids)) + tx.run( + ''' + MATCH (a:User {uid: $user_id}), (b:Resource {rid: $rid}) + MERGE (a)-[r:HAS_RATED {user_id: $user_id, rid: $rid}]->(b) + ON CREATE SET r.cids = $cids, r.value = $value + ON MATCH SET r.user_id = $user_id, r.rid = $rid, r.value = $value, r.cids = r.cids + [x IN $cids WHERE NOT x IN r.cids] + ''', + user_id=rating["user_id"], + rid=rating["rid"], + value=rating["value"], + cids=list(set(rating["cids"])) + ) + + # Update Resources: helpful_count and not_helpful_count + # WHERE size(r.cids) > 0 + count_helpful_count = tx.run( + ''' + MATCH p=(a: User)-[r:HAS_RATED {value: 'HELPFUL'}]->(b:Resource {rid: $rid}) + RETURN COUNT(r) AS count + ''', + rid=rating["rid"] + ).single() + + count_not_helpful_counter = tx.run( + ''' + MATCH p=(a: User)-[r:HAS_RATED {value: 'NOT_HELPFUL'}]->(b:Resource {rid: $rid}) + RETURN COUNT(r) AS count + ''', + rid=rating["rid"] + ).single() + + resource_has_rated_detail = tx.run( + ''' + MATCH (a:Resource {rid: $rid}) + SET a.helpful_count = $count_helpful_count, a.not_helpful_count = $count_not_helpful_counter + RETURN a.helpful_count as helpful_count, a.not_helpful_count as not_helpful_count + ''', + rid=rating["rid"], + count_helpful_count=count_helpful_count["count"], + count_not_helpful_counter=count_not_helpful_counter["count"] + ).single() + + result = { + "voted": rating["value"], + "helpful_count": resource_has_rated_detail["helpful_count"], + "not_helpful_count": resource_has_rated_detail["not_helpful_count"], + "reset_status": reset_status + } + return result + + def update_rs_btw_resource_and_cm(self, rid: str, cid: str, action=True): + ''' + Update by: Create and Delete Relationship + between Resource and Concept_modified + concepts: Concept list + cid: cid (from concepts) + action: True (create) | False (delete) + ''' + # logger.info("Updating Relationship between Resource and Concept_modified") + # print(rid, cid) + try: + tx = self.driver.session() + if action: + tx.run( + ''' + MATCH (u:Resource {rid: $rid}), (a:Concept_modified {cid: $cid}) + MERGE (u)-[r:BASED_ON]->(a) + RETURN r + ''', + rid=rid, + cid=cid + ) + else: + tx.run( + ''' + MATCH (u:Resource {rid: $rid})-[r:BASED_ON]->(a:Concept_modified {cid: $cid}) + DELETE r + ''', + rid=rid, + cid=cid + ) + except Exception as e: + print("update_rs_btw_resource_and_cm: Issue on this function either with rid or cid is None value") + print(e) + pass + + def update_rs_btw_resources_and_cm(self, rids: str, cid: str, action=True): + ''' + Update by: Create and Delete Relationship + between Resources and Concept_modified + concepts: Concept list + cid: cid (from concepts) + action: True (create) | False (delete) + ''' + # logger.info("Updating Relationship between Resource and Concept_modified") + try: + tx = self.driver.session() + if action: + tx.run( + ''' + MATCH (a:Concept_modified {cid: $cid}) + WITH a + MATCH (b:Resource) + WHERE b.rid IN $rids + MERGE (b)-[:BASED_ON]->(a) + ''', + rids=rids, + cid=cid + ) + except Exception as e: + print("update_rs_btw_resource_and_cm: Issue on this function either with rid or cid is None value") + print(e) + pass + + def user_saves_or_removes_resource(self, data: dict): + ''' + User saves or remove Resource(s) + data: { + "user_id": "vhf", + "mid": "dsdsd", + "slider_number": "slide_1", + "cid": "rewtg423", + "rid": "2gdsg", + "status": True (create) | False (remove) => (to create or remove a resource saved from the user list) + } + ''' + logger.info("Saving or Removing from Resource Saved List") + result = {"msg": ""} + tx = self.driver.session() + + if data["status"] == True: + tx.run( + ''' + MATCH (a:User {uid: $user_id}), (b:Resource {rid: $rid}) + MERGE (a)-[r:HAS_SAVED {user_id: $user_id, rid: $rid}]->(b) + ''', + user_id=data["user_id"], + rid=data["rid"] + ) + result["msg"] = "saved" + + else: + tx.run( + ''' + MATCH (a:User {uid: $user_id})-[r:HAS_SAVED {user_id: $user_id, rid: $rid}]->(b:Resource {rid: $rid}) + DELETE r + ''', + user_id=data["user_id"], + rid=data["rid"] + ) + result["msg"] = "removed" + + # Update Resources: saves_count + tx.run( + ''' + MATCH (a:Resource {rid: $rid}) + OPTIONAL MATCH (a)<-[r:HAS_SAVED {rid: $rid}]-() + WITH a, COUNT(r) AS saves_counter + SET a.saves_count = saves_counter + ''', + rid=data["rid"] + ) + logger.info("Saving or Removing from Resource Saved List: Done") + return result + + def store_resources(self, cid: str, resources_dict: dict=None, recommendation_type="", + resources_list: list=None, resources_form="dict", + resources_updated: list=None, resources_updated_type="" + ): + ''' + Store Resources + Create relationshop between Resource and Concept_modified + resources_dict: {"articles": [], "vidoes": []} + cid: str + recommendation_type: str ('1' | '2' | '3' | '4') + algorithm_model: (str) which algorithm was used for the recommendation + content_type: video | article # currently not usued + resources_form: dict | list | found (update_detail_found => only update too old Resource) + resources_updated_type: video | article + ''' + # if len(resources_dict["articles"]) > 0: + # print("store_resources ->") + # print(resources_dict["articles"][0]) + + + def get_resource_primary_key(resource: dict): + return resource["rid"] if "rid" in resource else resource["id"] + + logger.info("Store Resources: Videos | Articles") + # result = [] + + tx = self.driver.session() + if resources_form == "dict": + for key, resources in resources_dict.items(): + if key == "videos": + if len(resources) > 0: + logger.info(f"Creating Resources YouTube AND Updating Relationship between Resource and Concept_modified: {len(resources)} Resources") + for resource in resources: + self.create_or_update_video_resource(tx, resource, recommendation_type) + + rids = [get_resource_primary_key(resource) for resource in resources] + self.update_rs_btw_resources_and_cm(rids=rids, cid=cid, action=True) + + elif key == "articles": + if len(resources) > 0: + logger.info(f"Creating Resources Article AND Updating Relationship between Resource and Concept_modified {len(resources)} Resources") + for resource in resources: + self.create_or_update_wikipedia_resource(tx, resource, recommendation_type) + + rids = [get_resource_primary_key(resource) for resource in resources] + self.update_rs_btw_resources_and_cm(rids=rids, cid=cid, action=True) + + elif resources_form == "list": + for resource in resources_list: + if "Video" in resource["labels"]: + self.create_or_update_video_resource(tx, resource, update_embedding_values=True) + elif "Article" in resource["labels"]: + self.create_or_update_wikipedia_resource(tx, resource, update_embedding_values=True) + + """ + elif resources_form == "found": + if resources_updated_type == "videos": + for resource in resources_updated: + self.create_or_update_video_resource(tx, resource, update_detail_found=True) + if resources_updated_type == "articles": + for resource in resources_updated: + self.create_or_update_wikipedia_resource(tx, resource, update_detail_found=True) + """ + + + def retrieve_resources(self, concepts: dict, embedding_values=False): + ''' + Getting List of Resources connected to Concept_modified + algorithm_model: (str) which algorithm was used for the recommendation + query_form: + ''' + def resource_replace_none_value(value): + if value == None: + return 0 + return int(value) + + # logger.info("Getting List of Resources Containing Concept_modified") + + if embedding_values == True: + query = ''' + MATCH p=(a:Resource)-[r:BASED_ON]->(b:Concept_modified) + WHERE b.cid IN $cids + RETURN DISTINCT LABELS(a) as labels, ID(a) as id, a.rid as rid, a.title as title, a.text as text, + a.thumbnail as thumbnail, a.abstract as abstract, a.post_date as post_date, + a.author_image_url as author_image_url, a.author_name as author_name, + a.keyphrases as keyphrases, a.description as description, a.description_full as description_full, + a.publish_time as publish_time, a.uri as uri, a.duration as duration, + COALESCE(toInteger(a.views), 0) AS views, + COALESCE(toFloat(a.similarity_score), 0.0) AS similarity_score, + COALESCE(toInteger(a.helpful_count), 0) AS helpful_count, + COALESCE(toInteger(a.not_helpful_count), 0) AS not_helpful_count, + COALESCE(toInteger(a.bookmarked_count), 0) AS bookmarked_count, + COALESCE(toInteger(a.like_count), 0) AS like_count, + a.channel_title as channel_title, + a.updated_at as updated_at, + a.keyphrase_embedding as keyphrase_embedding, + a.document_embedding as document_embedding, + a.saves_count as saves_count, b.cid as concept_cid, + a.keyphrases_infos as keyphrases_infos + + ''' + else: + query = ''' + MATCH p=(a:Resource)-[r:BASED_ON]->(b:Concept_modified) + WHERE b.cid IN $cids + RETURN DISTINCT LABELS(a) as labels, ID(a) as id, a.rid as rid, a.title as title, a.text as text, + a.thumbnail as thumbnail, a.abstract as abstract, a.post_date as post_date, + a.author_image_url as author_image_url, a.author_name as author_name, + a.keyphrases as keyphrases, a.description as description, a.description_full as description_full, + a.publish_time as publish_time, a.uri as uri, a.duration as duration, + COALESCE(toInteger(a.views), 0) AS views, + COALESCE(toFloat(a.similarity_score), 0.0) AS similarity_score, + COALESCE(toInteger(a.helpful_count), 0) AS helpful_count, + COALESCE(toInteger(a.not_helpful_count), 0) AS not_helpful_count, + COALESCE(toInteger(a.bookmarked_count), 0) AS bookmarked_count, + COALESCE(toInteger(a.like_count), 0) AS like_count, + a.channel_title as channel_title, + a.updated_at as updated_at, + a.saves_count as saves_count, b.cid as concept_cid + ''' + + result = [] + cids = [node["cid"] for node in concepts] + with self.driver.session() as session: + result = session.run( + query, + cids=cids + ).data() + result = self.resources_wrapper_from_query(data=result) + return result + + def retrieve_resources_by_updated_at_exist_or_counts(self, cids, content_type: str, only_exist=True, days=30): + ''' + Validate Resource by checking whether it is too old or depending on the "days" given + cids: List[str] of concept_modified cid + content_type: Video | Article + days: 30 (Video) | 365 (Article) + ''' + + if content_type == "Article": + days = 365 + + if only_exist == True: + query = ''' + MATCH (a:Resource)-[r:BASED_ON]->(b:Concept_modified) + WHERE $content_type IN LABELS(a) AND b.cid IN $cids + RETURN COUNT(DISTINCT a) as count + ''' + else: + query = ''' + MATCH (a:Resource)-[r:BASED_ON]->(b:Concept_modified) + WHERE $content_type IN LABELS(a) AND b.cid IN $cids AND ( datetime(a.updated_at) > (datetime() - duration({days: $days})) ) + RETURN COUNT(DISTINCT a) as count + ''' + + count = 0 + with self.driver.session() as session: + result = session.run( + query, + cids=cids, + content_type=content_type, + days=days + ).single() + count = result["count"] + return count + + def resources_wrapper_from_query(self, data: list): + resources = [] + if data: + for resource in data: + # print([key for key, value in result[0].items() ]) + r = { + "id": resource["id"], + "title": resource["title"], + "rid": resource["rid"], + "uri": resource["uri"], + "helpful_count": resource["helpful_count"], + "not_helpful_count": resource["not_helpful_count"], + "labels": resource["labels"], + "similarity_score": resource["similarity_score"], + "keyphrases": resource["keyphrases"], + "text": resource["text"], + "bookmarked_count": resource["bookmarked_count"], + "updated_at": resource["updated_at"], + "keyphrase_embedding": resource["keyphrase_embedding"], + "document_embedding": resource["document_embedding"], + "is_bookmarked_fill": resource["is_bookmarked_fill"] if "is_bookmarked_fill" in resource else False, + "saves_count": resource["saves_count"] if "saves_count" in resource else 0, + "concept_cid": resource["concept_cid"] if "concept_cid" in resource else 0, + "keyphrases_infos": resource["keyphrases_infos"] + } + + if "Video" in r["labels"]: + r["description"] = resource["description"] + r["description_full"] = resource["description_full"] + r["thumbnail"] = resource["thumbnail"] + r["duration"] = resource["duration"] + r["views"] = resource["views"] + r["publish_time"] = resource["publish_time"] + r["like_count"] = resource["like_count"] + r["channel_title"] = resource["channel_title"] + + elif "Article" in r["labels"]: + r["abstract"] = resource["abstract"] + + resources.append(r) + return resources + + def update_resource_action(self, resource: dict, action=False): + ''' + Save or Update Resource Node from Ne4j + resource: resource detail + action: True (adding new resource) | + False (update the resource by attributes such as: similarity_score, views, like_count, channel_title) + ''' + + tx = self.driver.session() + if action: + if "Video" in resource["labels"]: + create_video_resource(tx=tx, node=resource) + elif "Article" in resource["labels"]: + create_wikipedia_resource(tx=tx, node=resource) + else: + # update + pass + + def filter_user_resources_saved_by(self, data: dict): + ''' + Getting User Resources Saved + Filtering by: user_id, cid: concept cid, mid: learning material and slide_number: silder number + data: { + cids: [], + user_id: 'assad83', + content_type: 'video | article', + text: 'neo4j node' + } + is_bookmarked_fill: default value: True because these resources belong to the user_id given + ''' + logger.info("Filtering User Resources Saved") + result = { "articles": [], "videos": [] } + query_where = """ + toLower(a.text) CONTAINS toLower($search_text) OR + ANY(keyphrase IN a.keyphrases WHERE keyphrase CONTAINS toLower($search_text)) + """ + + if len(data["cids"]) > 0: + query_where = """ + c.cid IN $cids AND ( toLower(a.text) CONTAINS toLower($search_text) OR + ANY(keyphrase IN a.keyphrases WHERE keyphrase CONTAINS toLower($search_text)) ) + """ + + with self.driver.session() as session: + nodes = session.run( + f""" + MATCH (c: Concept_modified)<-[m:HAS_MODIFIED]-(b: User)-[r:HAS_SAVED]->(a:Resource) + WHERE r.user_id = $user_id AND $content_type IN LABELS(a) AND ({query_where}) + RETURN DISTINCT LABELS(a) as labels, ID(a) as id, a.rid as rid, a.title as title, a.text as text, + a.thumbnail as thumbnail, a.abstract as abstract, a.post_date as post_date, + a.author_image_url as author_image_url, a.author_name as author_name, + a.keyphrases as keyphrases, a.description as description, a.description_full as description_full, + a.publish_time as publish_time, a.uri as uri, a.duration as duration, + COALESCE(toInteger(a.views), 0) AS views, + COALESCE(toFloat(a.similarity_score), 0.0) AS similarity_score, + COALESCE(toInteger(a.helpful_count), 0) AS helpful_count, + COALESCE(toInteger(a.not_helpful_count), 0) AS not_helpful_count, + COALESCE(toInteger(a.bookmarked_count), 0) AS bookmarked_count, + COALESCE(toInteger(a.like_count), 0) AS like_count, + a.channel_title as channel_title, + a.updated_at as updated_at, + true AS is_bookmarked_fill + """, + user_id=data["user_id"], + search_text=data["text"], + cids=data["cids"], + content_type= "Video" if data["content_type"] == "video" else "Article", + ).data() + resources = self.resources_wrapper_from_query(data=nodes) + + if len(resources) > 0: + if data["content_type"] == "video": + result["videos"] = resources + # result["videos"] = [resource for resource in resources if "Video" in resource["labels"]] + elif data["content_type"] == "article": + result["articles"] = resources + # result["articles"] = [resource for resource in resources if "Article" in resource["labels"]] + else: + result = { + "articles": [resource for resource in resources if "Article" in resource["labels"]], + "videos": [resource for resource in resources if "Video" in resource["labels"]] + } + return result + + def get_rids_from_user_saves(self, user_id: str): + ''' + Getting rids from User Resources Saved + ''' + logger.info("Getting rids from User Resources Saved") + nodes = [] + with self.driver.session() as session: + nodes = session.run( + ''' + MATCH (b:User)-[r:HAS_SAVED {user_id: $user_id}]->(a:Resource) + RETURN a.rid as rid + ''', + user_id=user_id + ).data() + nodes = [node["rid"] for node in nodes] + return nodes + + def get_main_concepts_by_mid(self, mid: str): + ''' + Getting main concepts by mid + ''' + logger.info("Getting main concepts by mid") + nodes = [] + with self.driver.session() as session: + nodes = session.run( + ''' + MATCH (c:Concept) + WHERE c.mid = $mid AND c.type = "main_concept" + RETURN ID(c) AS id, c.cid AS cid, c.name AS name, + c.type AS type, c.weight AS weight, c.mid AS mid + ORDER BY c.name + ''', + mid=mid + ).data() + return nodes + + def get_main_concepts_by_slide_id(self, slide_id: str): + ''' + Getting main concepts by mid + ''' + logger.info("Getting main concepts by mid") + nodes = [] + with self.driver.session() as session: + nodes = session.run( + ''' + MATCH p=(s: Slide)-[r: CONSISTS_OF]->(c: Concept) + WHERE s.sid = $slide_id AND c.type = "main_concept" + RETURN ID(c) as id, c.cid as cid, c.name as name, + c.type AS type, c.weight AS weight, c.mid AS mid + ORDER BY c.name DESC + ''', + slide_id=slide_id + ).data() + return nodes + # TODO Issue #640: do we need create_user_slide_relationships? diff --git a/coursemapper-kg/recommendation/app/services/course_materials/kwp_extraction/dbpedia/data_service1.py b/coursemapper-kg/recommendation/app/services/course_materials/kwp_extraction/dbpedia/data_service1.py index de0f9a5fc..8e202cf9e 100644 --- a/coursemapper-kg/recommendation/app/services/course_materials/kwp_extraction/dbpedia/data_service1.py +++ b/coursemapper-kg/recommendation/app/services/course_materials/kwp_extraction/dbpedia/data_service1.py @@ -52,7 +52,7 @@ def _get_concept_recommendation(self, user_id, mid): recommend_concepts = self.recommendation.recommend(concept_list, user, top_n=5) for i in recommend_concepts: info = i["n"]["name"] + " : " + str(i["n"]["score"]) - logger.info(info) + # logger.info(info) # Use paths for interpretability recommend_concepts = self._get_road(recommend_concepts, user_id, mid) @@ -108,8 +108,8 @@ def get_max_weight_path(self, road): names.append(road[i]["name"]) for name in names: for i in range(len(road)): - print("len(road)",len(road)) - print("name",name) + # print("len(road)",len(road)) + # print("name",name) if road[i]["name"] == name: weights = road[i]["weight"] if max_weight <= weights: diff --git a/coursemapper-kg/recommendation/app/services/course_materials/recommendation/recommendation_type.py b/coursemapper-kg/recommendation/app/services/course_materials/recommendation/recommendation_type.py index 14dacbfc5..a63d45143 100644 --- a/coursemapper-kg/recommendation/app/services/course_materials/recommendation/recommendation_type.py +++ b/coursemapper-kg/recommendation/app/services/course_materials/recommendation/recommendation_type.py @@ -3,9 +3,45 @@ class RecommendationType(Enum): WITHOUT_EMBEDDING = "without_embedding" - STATIC_DOCUMENT_BASED = "static_document_based" - STATIC_KEYPHRASE_BASED = "static_keyphrase_based" - DYNAMIC_KEYPHRASE_BASED = "dynamic_keyphrase_based" - DYNAMIC_DOCUMENT_BASED = "dynamic_document_based" + CONTENT_BASED_DOCUMENT_VARIANT = "static_document_based" + CONTENT_BASED_KEYPHRASE_VARIANT = "static_keyphrase_based" + PKG_BASED_KEYPHRASE_VARIANT = "dynamic_keyphrase_based" + PKG_BASED_DOCUMENT_VARIANT = "dynamic_document_based" COMBINED_DYNAMIC = "combined_dynamic" COMBINED_STATIC = "combined_static" + + def map_type(recommendation_type: str, find_type = "k"): + """ + Get key or Value: + recommendation_type: RecommendationType.PKG_BASED_KEYPHRASE_VARIANT, + find_type: v = value and k = key + """ + sources = { + "1": RecommendationType.PKG_BASED_KEYPHRASE_VARIANT, + "2": RecommendationType.PKG_BASED_DOCUMENT_VARIANT, + "3": RecommendationType.CONTENT_BASED_KEYPHRASE_VARIANT, + "4": RecommendationType.CONTENT_BASED_DOCUMENT_VARIANT, + } + + if find_type == "v": + value = None + for k, v in sources.items(): + if recommendation_type == v: + value = k + break + else: + value = sources[recommendation_type] + + return value + + def map_type2(recommendation_type: str): + if recommendation_type == "1": + result = RecommendationType.PKG_BASED_KEYPHRASE_VARIANT + elif recommendation_type == "2": + result = RecommendationType.PKG_BASED_DOCUMENT_VARIANT + elif recommendation_type == "3": + result = RecommendationType.CONTENT_BASED_KEYPHRASE_VARIANT + elif recommendation_type == "4": + result = RecommendationType.CONTENT_BASED_DOCUMENT_VARIANT + + return result diff --git a/coursemapper-kg/recommendation/app/services/course_materials/recommendation/recommender.py b/coursemapper-kg/recommendation/app/services/course_materials/recommendation/recommender.py index c3300eee6..c6444fbf7 100644 --- a/coursemapper-kg/recommendation/app/services/course_materials/recommendation/recommender.py +++ b/coursemapper-kg/recommendation/app/services/course_materials/recommendation/recommender.py @@ -11,6 +11,7 @@ from ..kwp_extraction.dbpedia.concept_tagging import DBpediaSpotlight from .wikipedia_service import WikipediaService from .youtube_service import YoutubeService +import json from log import LOG @@ -21,16 +22,19 @@ def compute_combined_similatity(data, alpha, recommendation_type, with_user): logger.info("Compute Fused Cosine Similarities") if with_user: data[recommendation_type] = (1 - alpha) * data[ - RecommendationType.DYNAMIC_DOCUMENT_BASED - ] + data[RecommendationType.DYNAMIC_KEYPHRASE_BASED] * alpha + RecommendationType.PKG_BASED_DOCUMENT_VARIANT + ] + data[RecommendationType.PKG_BASED_KEYPHRASE_VARIANT] * alpha else: data[recommendation_type] = (1 - alpha) * data[ - RecommendationType.STATIC_DOCUMENT_BASED - ] + data[RecommendationType.STATIC_KEYPHRASE_BASED] * alpha + RecommendationType.CONTENT_BASED_DOCUMENT_VARIANT + ] + data[RecommendationType.CONTENT_BASED_KEYPHRASE_VARIANT] * alpha return data - def sort_by_similarity_type(data, similarity_type): + ''' + sort results by similarity score + sorted_data = sort_by_similarity_type(data, recommendation_type) + ''' logger.info("Sort Results") sorted_data = data.sort_values( @@ -39,150 +43,107 @@ def sort_by_similarity_type(data, similarity_type): return sorted_data - def compute_cosine_similarity_with_embeddings(embedding1, embedding2): return util.pytorch_cos_sim(embedding1, embedding2).item() -def compute_dynamic_document_based_similarity( - data, recommendation_type, user_embedding -): - cosine_similarities = [] - - for document_embedding in data["document_embedding"]: - tensor = get_tensor_from_embedding(document_embedding) - - cosine_similarity = compute_cosine_similarity_with_embeddings( - tensor, user_embedding - ) - cosine_similarities.append(cosine_similarity) - - data[recommendation_type] = cosine_similarities - - return data - - def get_tensor_from_embedding(embedding): + ''' + Tranform embedding to tensor + ''' embedding_array = [float(i) for i in embedding] embedding_array = np.array(embedding_array) embedding_tensor = torch.from_numpy(embedding_array).float() return embedding_tensor -def compute_dynamic_keyphrase_based_similarity( - data, recommendation_type, user_embedding -): - cosine_similarities = [] - - for keyphrase_embedding in data["keyphrase_embedding"]: - embedding_array = [float(i) for i in keyphrase_embedding] - embedding_array = np.array(embedding_array) - tensor = torch.from_numpy(embedding_array).float() - - cosine_similarity = compute_cosine_similarity_with_embeddings( - tensor, user_embedding - ) - cosine_similarities.append(cosine_similarity) - - data[recommendation_type] = cosine_similarities - - return data +def compute_similarity(embedding_type: str, data, embedding_tensor): + ''' + Compute Cosine Similarities + ''' + logger.info(f"Computing Cosine Similarities with {embedding_type}") - -def compute_document_based_similarity( - data, slide_document_embedding, recommendation_type -): - logger.info("Compute Document-Based Cosine Similarities") cosine_similarities = [] - - slide_document_embedding_array = slide_document_embedding.split(",") - slide_document_embedding_tensor = get_tensor_from_embedding( - slide_document_embedding_array - ) - - for document_embedding in data["document_embedding"]: - tensor = get_tensor_from_embedding(document_embedding) + for embedding in data[embedding_type]: + tensor = get_tensor_from_embedding(embedding) cosine_similarity = compute_cosine_similarity_with_embeddings( - slide_document_embedding_tensor, tensor + tensor, embedding_tensor ) cosine_similarities.append(cosine_similarity) - - data[recommendation_type] = cosine_similarities - + data["similarity_score"] = cosine_similarities return data def retrieve_keyphrases(data): + ''' + Retrieve keyphrases + ''' logger.info("Add relevant columns for keyphrases") - resource_keyphrases_infos = [] - resource_keyphrases = [] - keyphrase_counts = [] - - logger.info(data) - for index, text in enumerate(data["text"]): - pos = {"NOUN", "PROPN", "ADJ"} - extractor = SingleRank() - extractor.load_document(input=text, language="en") - extractor.candidate_selection(pos=pos) - extractor.candidate_weighting(window=10, pos=pos) - keyphrases_infos = extractor.get_n_best(n=15) - keyphrases = [keyphrase[0] for keyphrase in keyphrases_infos] - - keyphrase_counts.append(len(keyphrases)) - resource_keyphrases.append(keyphrases) - resource_keyphrases_infos.append(keyphrases_infos) - index += 1 - data["keyphrase_counts"] = keyphrase_counts - data["keyphrases"] = resource_keyphrases - data["keyphrases_infos"] = resource_keyphrases_infos + for index, row in data.iterrows(): + if len(row["keyphrases"]) < 1: + text = row["text"] + pos = {"NOUN", "PROPN", "ADJ"} + extractor = SingleRank() + extractor.load_document(input=text, language="en") + extractor.candidate_selection(pos=pos) + extractor.candidate_weighting(window=10, pos=pos) + keyphrases_infos = extractor.get_n_best(n=15) + keyphrases = [keyphrase[0] for keyphrase in keyphrases_infos] + + keyphrases_infos_modified = [list(item) for item in keyphrases_infos] + data.at[index, 'keyphrases_infos'] = json.dumps(keyphrases_infos_modified) + # data.at[index, 'keyphrases_infos'] = keyphrases_infos + data.at[index, 'keyphrases'] = keyphrases return data - class Recommender: def __init__(self, embedding_model): self.youtube_service = YoutubeService() self.wikipedia_service = WikipediaService() self.dbpedia = DBpediaSpotlight() if not embedding_model: - embedding_model = "sentence-transformers/msmarco-distilbert-base-tas-b" + # embedding_model = "sentence-transformers/msmarco-distilbert-base-tas-b" + embedding_model = "all-mpnet-base-v2" self.embedding = SentenceTransformer(embedding_model) - def canditate_selection(self, query, video): + def canditate_selection(self, query, video, result_type="records", top_n_videos=2, top_n_articles=2): + ''' + query: string to query API content + video: content type with True for video and False for wikipedia content + result_type: which form to deliver the resources crawled: records (list of dict) + top_n_videos: number of content to get from YouTube API + top_n_articles: number of content to get from Wikipedia API + ''' data: pd.DataFrame + # top_n = 2 # 15 if video: start_time = time.time() - data = self.youtube_service.get_videos(query) + data = self.youtube_service.get_videos(query, top_n=top_n_videos) end_time = time.time() print("Get Videos Execution time: ", end_time - start_time, flush=True) else: start_time = time.time() - data = self.wikipedia_service.get_articles(query) + data = self.wikipedia_service.get_articles(query, top_n=top_n_articles) end_time = time.time() print("Get Articles Execution time: ", end_time - start_time, flush=True) + if result_type == "records": + return data.to_dict('records') return data - def recommend( - self, - not_understood_concept_list, - slide_concepts=[], - slide_weighted_avg_embedding_of_concepts="", - slide_document_embedding="", - user_embedding="", - top_n=10, - video=True, - recommendation_type=RecommendationType.WITHOUT_EMBEDDING, - ): + def _get_data(self, recommendation_type, not_understood_concept_list, slide_concepts, video): # If personalized recommendation, use DNU concepts to query Youtube and Wikipedia if ( recommendation_type != RecommendationType.WITHOUT_EMBEDDING and recommendation_type != RecommendationType.COMBINED_STATIC - and recommendation_type != RecommendationType.STATIC_KEYPHRASE_BASED - and recommendation_type != RecommendationType.STATIC_DOCUMENT_BASED + and recommendation_type != RecommendationType.CONTENT_BASED_KEYPHRASE_VARIANT + and recommendation_type != RecommendationType.CONTENT_BASED_DOCUMENT_VARIANT ): + logger.info("# If personalized recommendation, use DNU concepts to query Youtube and Wikipedia") + query = " ".join(not_understood_concept_list) data = self.canditate_selection(query=query, video=video) if isinstance(data, list) and data.empty: @@ -190,6 +151,8 @@ def recommend( # Else use top 5 concepts from slide else: + logger.info("# Else use top 5 concepts from slide") + i = 0 while i < 5: top_n_concepts = 5 - i @@ -206,295 +169,147 @@ def recommend( if len(data.index) >= 5: break i += 1 - # Without embedding Not yet supported - if recommendation_type == RecommendationType.WITHOUT_EMBEDDING: - return data - # if Model 4 - elif recommendation_type == RecommendationType.STATIC_DOCUMENT_BASED: - start_time = time.time() - - # Step 1: compute document embedding for resources - data = self.compute_document_based_embeddings(data) - - end_time = time.time() - print( - "Retrieve term-based embeddings Execution time: ", - end_time - start_time, - flush=True, - ) - start_time = time.time() - - # Step 2: compute similarities between slide document embeddings and resources document embeddings - logger.info("Compute Cosine Similarities") - data = compute_document_based_similarity( - data, slide_document_embedding, recommendation_type - ) - - end_time = time.time() - print( - "Compute term-based similarity Execution time: ", - end_time - start_time, - flush=True, - ) - - start_time = time.time() - - # Step 3: sort results - sorted_data = sort_by_similarity_type(data, recommendation_type) - - end_time = time.time() - - print( - "Compute term-based Sort Execution time: ", - end_time - start_time, - flush=True, - ) - - return sorted_data.head(top_n) - # If Model 3 - elif recommendation_type == RecommendationType.STATIC_KEYPHRASE_BASED: - start_time = time.time() - - # Step 1: Retrieve Keyphrases - data = retrieve_keyphrases(data) - - end_time = time.time() - print( - "Retrieve keyphrases Execution time: ", - end_time - start_time, - flush=True, - ) - - start_time = time.time() - - # Step 2: compute keyphrase-based embedding for resources - data = self.compute_keyphrase_based_embeddings(data) - end_time = time.time() - print( - "Retrieve keyphrase-based embedding Execution time: ", - end_time - start_time, - flush=True, - ) - start_time = time.time() - # Step 3: compute keyphrase-based similarity between slide weighted average keyphrase embeddings and - # resources weighted average keyphrase embeddings - logger.info("Compute Cosine Similarities") - data = compute_keyphrase_based_similarity( - data, slide_weighted_avg_embedding_of_concepts, recommendation_type - ) - end_time = time.time() - print( - "Retrieve keyphrase-based embedding Execution time: ", - end_time - start_time, - flush=True, - ) - start_time = time.time() - - # Step 4: sort results by similarity score - sorted_data = sort_by_similarity_type(data, recommendation_type) - - end_time = time.time() - print( - "Retrieve keyphrase-based embeddings Execution time: ", - end_time - start_time, - flush=True, - ) + if data.empty == False: + logger.info(f"canditate_selection shape -> {data.shape}") + return data - return sorted_data.head(top_n) + def recommend( + self, + recommendation_type=RecommendationType.WITHOUT_EMBEDDING, + data:pd.DataFrame=None, + slide_weighted_avg_embedding_of_concepts="", + slide_document_embedding="", + user_embedding="", + top_n=10, + video=True, + not_understood_concept_list=[], + slide_concepts=[], + ): + ''' + Apply recommendation algorithms + data: resources in DataFrame + are_embedding_values_present: False (not empty) | True (empty) ( + if the resource contains key values: keyphrase_embedding | document_embedding) + ''' + logger.info("Applying the recommendation algorithm Selected") # Model 1 - elif recommendation_type == RecommendationType.DYNAMIC_KEYPHRASE_BASED: + if recommendation_type == RecommendationType.PKG_BASED_KEYPHRASE_VARIANT: + logger.info("Algorithm Model 1: Starting Processing") start_time = time.time() - + # Tranform embedding to tensor - user_embedding_array = user_embedding.split(",") - user_tensor = get_tensor_from_embedding(user_embedding_array) + user_tensor = get_tensor_from_embedding(user_embedding.split(",")) # Step 1: Retrieve Keyphrases data = retrieve_keyphrases(data) - end_time = time.time() - print( - "Retrieve keyphrases Execution time: ", - end_time - start_time, - flush=True, - ) - - start_time = time.time() # Step 2: compute keyphrase-based embedding for resources data = self.compute_keyphrase_based_embeddings(data) - end_time = time.time() - print( - "Retrieve keyphrase-based embeddings Execution time: ", - end_time - start_time, - flush=True, - ) - - start_time = time.time() - + # Step 3: compute keyphrase-based similarity between user embeddings and # resources weighted average keyphrase embeddings - logger.info("Compute Cosine Similarities") - data = compute_dynamic_keyphrase_based_similarity( - data, recommendation_type, user_embedding=user_tensor - ) - end_time = time.time() - print( - "Retrieve keyphrase-based similarity Execution time: ", - end_time - start_time, - flush=True, - ) - - start_time = time.time() - - # Step 4: sort results by similarity score - sorted_data = sort_by_similarity_type(data, recommendation_type) - - end_time = time.time() - print( - "Retrieve keyphrase-based sorted data Execution time: ", - end_time - start_time, - flush=True, - ) - - return sorted_data.head(top_n) + data = compute_similarity(embedding_type="keyphrase_embedding", data=data, embedding_tensor=user_tensor) + + total_time = time.time() - start_time + logger.info(f"Algorithm Model 1: Execution time {str(total_time)}") + return data + # If Model 2 - elif recommendation_type == RecommendationType.DYNAMIC_DOCUMENT_BASED: - # Transform embedding to tensor - user_embedding_array = user_embedding.split(",") - user_embedding_array = [float(i) for i in user_embedding_array] - user_embedding_array = np.array(user_embedding_array) - user_tensor = torch.from_numpy(user_embedding_array).float() + elif recommendation_type == RecommendationType.PKG_BASED_DOCUMENT_VARIANT: start_time = time.time() + # Transform embedding to tensor + user_tensor = get_tensor_from_embedding(user_embedding.split(",")) + # Step 1: compute document embedding for resources data = self.compute_document_based_embeddings(data) - end_time = time.time() - print( - "Retrieve term-based embeddings Execution time: ", - end_time - start_time, - flush=True, - ) - start_time = time.time() - + # Step 2: compute similarities between user embeddings and resources document embeddings - logger.info("Compute Cosine Similarities") - data = compute_dynamic_document_based_similarity( - data, recommendation_type, user_embedding=user_tensor - ) - end_time = time.time() - print( - "Retrieve user document-based similarity Execution time: ", - end_time - start_time, - flush=True, - ) - start_time = time.time() - - # Step 3: sort results - sorted_data = sort_by_similarity_type(data, recommendation_type) + data = compute_similarity(embedding_type="document_embedding", data=data, embedding_tensor=user_tensor) - end_time = time.time() - print("Sort result Execution time: ", end_time - start_time, flush=True) - - return sorted_data.head(top_n) - # If Combined Dynamic Model. - # TODO this model was Ignored during the evalution. Can be considered in future works - elif recommendation_type == RecommendationType.COMBINED_DYNAMIC: + total_time = time.time() - start_time + logger.info(f"Algorithm Model 2: Execution time {str(total_time)}") + return data + + # If Model 3 + elif recommendation_type == RecommendationType.CONTENT_BASED_KEYPHRASE_VARIANT: start_time = time.time() - user_embedding_array = user_embedding.split(",") - user_embedding_array = [float(i) for i in user_embedding_array] - user_embedding_array = np.array(user_embedding_array) - user_tensor = torch.from_numpy(user_embedding_array).float() + # Tranform embedding to tensor + slide_weighted_avg_embedding_of_concepts_embedding = get_tensor_from_embedding(slide_weighted_avg_embedding_of_concepts) + # Step 1: Retrieve Keyphrases data = retrieve_keyphrases(data) - data = self.compute_keyphrase_based_embeddings(data) - data = self.compute_document_based_embeddings(data) - - logger.info("Compute Cosine Similarities") - data = compute_dynamic_keyphrase_based_similarity( - data, recommendation_type, user_embedding=user_tensor - ) - data = compute_dynamic_document_based_similarity( - data, recommendation_type, user_embedding=user_tensor - ) - data = compute_combined_similatity( - data, 0.5, recommendation_type, with_user=True - ) - sorted_data = sort_by_similarity_type(data, recommendation_type) + # Step 2: compute keyphrase-based embedding for resources + data = self.compute_keyphrase_based_embeddings(data) - end_time = time.time() - print( - "Retrieve combined similarity with user embeddings Execution time: ", - end_time - start_time, - flush=True, - ) + # Step 3: compute keyphrase-based similarity between slide weighted average keyphrase embeddings and + # resources weighted average keyphrase embeddings + data = compute_similarity(embedding_type="keyphrase_embedding", data=data, embedding_tensor=slide_weighted_avg_embedding_of_concepts_embedding) - return sorted_data.head(top_n) + total_time = time.time() - start_time + logger.info(f"Algorithm Model 3: Execution time {str(total_time)}") + return data # sorted_data.head(top_n) - # If Combined Static Model. - # TODO Ignored during the evalution. Can be considered in future works - elif recommendation_type == RecommendationType.COMBINED_STATIC: + # if Model 4 + elif recommendation_type == RecommendationType.CONTENT_BASED_DOCUMENT_VARIANT: + # Compute term-based start_time = time.time() - data = retrieve_keyphrases(data) - data = self.compute_keyphrase_based_embeddings(data) - data = self.compute_document_based_embeddings(data) + # Transform embedding to tensor + slide_document_embedding_tensor = get_tensor_from_embedding(slide_document_embedding.split(",")) - logger.info("Compute Cosine Similarities") - data = compute_document_based_similarity( - data, slide_document_embedding, recommendation_type - ) - data = self.slide_weighted_avg_embedding_of_concepts( - data, slide_weighted_avg_embedding_of_concepts, recommendation_type - ) - data = compute_combined_similatity( - data, 0.5, recommendation_type, with_user=False - ) + # Step 1: compute document embedding for resources + data = self.compute_document_based_embeddings(data) - sorted_data = sort_by_similarity_type(data, recommendation_type) + # Step 2: compute similarities between slide document embeddings and resources document embeddings + data = compute_similarity(embedding_type="document_embedding", data=data, embedding_tensor=slide_document_embedding_tensor) - end_time = time.time() - print( - "Retrieve combined similarity without user embeddings Execution time: ", - end_time - start_time, - flush=True, - ) + total_time = time.time() - start_time + logger.info(f"Algorithm Model 4: Execution time {str(total_time)}") + return data - return sorted_data.head(top_n) def compute_keyphrase_based_embeddings(self, data): - logger.info("Add relevant Columns for keyphrase-based embeddings") - - resource_keyphrase_embeddings = [] - - for index, keyphrase_infos in enumerate(data["keyphrases_infos"]): - keyphrases_avg_embedding = ( - self.compute_weighted_avg_embedding_of_keyphrases( - keyphrase_infos - ).tolist() - ) - - resource_keyphrase_embeddings.append(keyphrases_avg_embedding) - - data["keyphrase_embedding"] = resource_keyphrase_embeddings - + ''' + Compute keyphrase-based embedding for resources + ''' + logger.info("Add relevant Columns for keyphrase embeddings") + # print(data["keyphrases_infos"].head(5)) + + def do(row): + if len(row["keyphrase_embedding"]) < 1: + keyphrase_infos = row["keyphrases_infos"] + keyphrases_avg_embedding = ( + self.compute_weighted_avg_embedding_of_keyphrases( + keyphrase_infos + ).tolist() + ) + return keyphrases_avg_embedding + else: + return row["keyphrase_embedding"] + + data['keyphrase_embedding'] = data.apply(do, axis=1) return data def compute_document_based_embeddings(self, data): + ''' + Compute document embedding for resources + Retrieve term-based embeddings + ''' logger.info("Add relevant Columns for document embeddings") - resource_document_based_embeddings = [] - - for index, text in enumerate(data["text"]): - logger.info(text) - embedding = self.embedding.encode(text) - - resource_document_based_embeddings.append(embedding) - - data["document_embedding"] = resource_document_based_embeddings - + def do(row): + if len(row["document_embedding"]) < 1: + text = row["text"] + value = self.embedding.encode(text) + return value.tolist() + else: + return row["document_embedding"] + + data['document_embedding'] = data.apply(do, axis=1) return data def compute_cosine_similarity_with_text(self, text1, text2): @@ -521,17 +336,18 @@ def compute_weighted_avg_embedding_of_keyphrases(self, keyphrase_infos): np.seterr("raise") embedding_list = [] weight_sum = 0 + keyphrase_infos = json.loads(keyphrase_infos) for keyphrase in keyphrase_infos: embedding = self.embedding.encode(keyphrase[0]) embedding_list.append(embedding * keyphrase[1]) weight_sum += keyphrase[1] if len(embedding_list) != 0: vectors = np.sum(embedding_list, axis=0) - print("Lenght: %s", len(embedding_list)) - print("Weight Sum: %s", weight_sum) + # print("Lenght: %s", len(embedding_list)) + # print("Weight Sum: %s", weight_sum) # dividing by the sum of weights average_embedding = vectors / weight_sum - print("average_embedding: %s", type(average_embedding)) + # print("average_embedding: %s", type(average_embedding)) return average_embedding else: return 0 diff --git a/coursemapper-kg/recommendation/app/services/course_materials/recommendation/resource_recommender.py b/coursemapper-kg/recommendation/app/services/course_materials/recommendation/resource_recommender.py index d53bf5ffa..fe3194caa 100644 --- a/coursemapper-kg/recommendation/app/services/course_materials/recommendation/resource_recommender.py +++ b/coursemapper-kg/recommendation/app/services/course_materials/recommendation/resource_recommender.py @@ -1,11 +1,18 @@ -import concurrent.futures - from ..db.neo4_db import NeoDataBase from .recommendation_type import RecommendationType from .recommender import Recommender +import numpy as np from config import Config +import numpy as np +import pandas as pd + +from ..recommendation import resource_recommender_helper as rrh + +from log import LOG +import logging +logger = LOG(name=__name__, level=logging.DEBUG) class ResourceRecommenderService: def __init__(self): @@ -24,6 +31,11 @@ def check_parameters( new_concept_ids, recommendation_type, ): + ''' + Check if parameters exist. If one doesn't exist, return not found message + check_message = resource_recommender_service.check_parameters( + slide_id, material_id, user_id, non_understood_concept_ids, understood_concept_ids, new_concept_ids, recommendation_type) + ''' if not self.db.slide_exists(slide_id): return "No Slide found with id: %s" % slide_id @@ -57,6 +69,31 @@ def _construct_user(self, user_id, non_understood, understood, new_concepts, mid user_id, non_understood, understood, new_concepts, mid ) + def resources_crawler_logic(self, concepts, recommendation_type=None): + results = [] + recommender = Recommender(embedding_model=None) + + # Check if concepts already exist and connected to any resources in Neo4j Database + concepts_db_checked = rrh.check_and_validate_resources(db=self.db, concepts=concepts) + + # Crawl resources from YouTube (from each dnu) and Wikipedia API + for concept_updated in concepts_db_checked: #i in range(len(not_understood_concept_list)): + results.append(rrh.parallel_crawling_resources(function=recommender.canditate_selection, + concept_updated=concept_updated, + result_type="records", + top_n_videos=10, + top_n_articles=10 + )) + + if recommendation_type: + # Store resources into Neo4j Database (by creating connection btw Resource and Concept_modified) + for result in results: + self.db.store_resources(resources_dict=result, cid=result["cid"], + recommendation_type=recommendation_type, + resources_form="dict" + ) + + def _get_personalized_recommendation( self, not_understood_concept_list, @@ -88,257 +125,139 @@ def _get_static_recommendation( recommendation_type=recommendation_type, ) - def _get_resources(self, user_id, slide_id, material_id, recommendation_type): - """ """ - wikipedia_articles = [] - youtube_videos = [] - resource_list = [] - relationship_list = [] - not_understood_concept_list = [] - concepts = [] - - material = self.db.get_lm(material_id) - embedding_model = None - if len(material) > 0 and "embedding_model" in material[0]["m"]: - embedding_model = material[0]["m"]["embedding_model"] - - self.recommender = Recommender(embedding_model=embedding_model) - - # Allow parallel recommendation of videos and articles - with concurrent.futures.ThreadPoolExecutor() as executor: - futures = {} - resource_types = ["Youtube", "Wikipedia"] - - # If personalized, get user information from the database then proceed with the personalized recommendation - if ( - recommendation_type != RecommendationType.WITHOUT_EMBEDDING - and self.db.user_exists(user_id) - and recommendation_type != RecommendationType.COMBINED_STATIC - and recommendation_type != RecommendationType.STATIC_KEYPHRASE_BASED - and recommendation_type != RecommendationType.STATIC_DOCUMENT_BASED - ): - concepts = self.db.get_top_n_dnu_concepts( - user_id=user_id, material_id=material_id, top_n=5 - ) - user_embedding = self.db.get_or_create_user(user_id)[0]["u"][ - "embedding" - ] - not_understood_concept_list = [concept["name"] for concept in concepts] - concept_ids = [concept["cid"] for concept in concepts] - - for resource_type in resource_types: - # Start the load operations and mark each future with its URL - future = executor.submit( - self._get_personalized_recommendation, - not_understood_concept_list=not_understood_concept_list, - user_embedding=user_embedding, - resource_type=resource_type, - recommendation_type=recommendation_type, - ) - futures[future] = resource_type - # Else retrieve Slide information from the database then proceed with the static recommendation - else: - _slide = self.db.get_slide(slide_id) - slide_concepts = _slide[0]["s"]["concepts"] - # slide_text = _slide[0]["s"]["text"] - slide_document_embedding = _slide[0]["s"]["initial_embedding"] - slide_weighted_avg_embedding_of_concepts = _slide[0]["s"][ - "weighted_embedding_of_concept" - ] - - for name in slide_concepts: - concepts.append(self.db.get_top_n_concept_by_name(name=name)[0]) - - concept_ids = [concept["cid"] for concept in concepts] - - for resource_type in resource_types: - # Start the load operations and mark each future with the resource type - future = executor.submit( - self._get_static_recommendation, - slide_document_embedding=slide_document_embedding, - slide_concepts=concepts, - slide_weighted_avg_embedding_of_concepts=slide_weighted_avg_embedding_of_concepts, - resource_type=resource_type, - recommendation_type=recommendation_type, - ) - futures[future] = resource_type - - # When one of the parallel operations is finish retrieve results - # 3000s is the maximum time allowed for each operation - for future in concurrent.futures.as_completed(futures, 3000): - print("future value") - print(future) - data_type = futures[future] - try: - if data_type == "Youtube": - youtube_videos = future.result() - print(youtube_videos) - else: - wikipedia_articles = future.result() - print(wikipedia_articles) - except Exception as exc: - print("%r generated an exception: %s" % (data_type, exc)) - - # for cid in concept_ids: - # - # resources, relationships = self.db.get_concept_resources(cid, user_id) - # - # if not resources: - # if self.db.concept_exists(cid): - # concept_name = self.db.get_concept_name_by_id(cid)[0]['c']['name'] - # not_understood_concept_list.append(concept_name) - # else: - # else: - # resource_list.extend([r for r in resources if r not in resource_list]) - # relationship_list.extend([r for r in relationships if r not in relationship_list]) - # - # if len(not_understood_concept_list) <= 0 < len(resource_list): - # - # return get_serialized_resource_data(resource_list, relationship_list) - - # If both results are empty return an empty object - if (isinstance(youtube_videos, list) or youtube_videos.empty) and ( - isinstance(wikipedia_articles, list) or wikipedia_articles.empty + + def process_recommandation_pipeline(self, + rec_params: dict, + factor_weights: dict, + recommendation_type, + user_embedding="", + slide_weighted_avg_embedding_of_concepts="", + slide_document_embedding="", + top_n_resources=10 ): - return {} - - # Otherwise proceed save the results in the database - resources, relationships = self.db.get_or_create_resoures_relationships( - wikipedia_articles=wikipedia_articles, - youtube_videos=youtube_videos, - user_id=user_id, - material_id=material_id, - concept_ids=concept_ids, + recommender = Recommender(embedding_model=None) + self.resources_crawler_logic(concepts=rec_params["concepts"], recommendation_type=rec_params["recommendation_type"]) + + # Gather|Retrieve all resources crawled + resources = self.db.retrieve_resources(concepts=rec_params["concepts"], embedding_values=True) + logger.info(f"len of resources {len(resources)}") + + # process with the recommendation algorithm selected + if len(resources) > 0: + data_df = pd.DataFrame(resources) + resources_df = recommender.recommend( + slide_weighted_avg_embedding_of_concepts=slide_weighted_avg_embedding_of_concepts, + slide_document_embedding=slide_document_embedding, + user_embedding=user_embedding, + top_n=10, + recommendation_type=recommendation_type, + data=data_df + ) + resources_df.replace({np.nan: None}, inplace=True) + resources = resources_df.to_dict(orient='records') + self.db.store_resources(resources_list=resources, resources_form="list",resources_dict=None, cid=None) + + # insert attribute "is_bookmarked_fill" for resource saved by the user + rids_user_resources_saved = self.db.get_rids_from_user_saves(user_id=rec_params["user_id"]) + resources = [{**resource, 'is_bookmarked_fill': resource['rid'] in rids_user_resources_saved} for resource in resources] + + # Apply ranking algorithm on the resources + resources_dict = rrh.rank_resources(resources=resources, weights=factor_weights, + top_n_resources=top_n_resources, recommendation_type=recommendation_type, + pagination_params=rec_params["pagination_params"] + ) + result_final = { + "recommendation_type": rec_params["recommendation_type"], + "concepts": rec_params["concepts"], + "nodes": resources_dict + } + return result_final + + def _get_resources(self, data_rec_params: dict, data_default: dict=None, top_n = 5): + ''' + Save cro_form, Crawl Youtube and Wikipedia API and then Store Resources + Result: { "recommendation_type": "", "concepts": [], "nodes": {"articles": [], "videos": []} } + ''' + body = rrh.rec_params_request_mapped(data_rec_params, data_default) + result = { "recommendation_type": "", "concepts": [], "nodes": {"articles": [], "videos": []} } + + # Map recommendation type to enum values + rec_params = body["rec_params"] + # recommendation_type_str = body["rec_params"]["recommendation_type"] + recommendation_type = RecommendationType.map_type(rec_params["recommendation_type"]) + + check_message = self.check_parameters( + slide_id=body["slide_id"], + material_id=body["material_id"], + non_understood_concept_ids=body["non_understood_concept_ids"], + understood_concept_ids=body["understood_concept_ids"], + new_concept_ids=body["new_concept_ids"], + recommendation_type=rec_params["recommendation_type"], + ) + if check_message != "": + return result + + # user = {"user_id": body["user_id"], "name": body["username"], "user_email": body["user_email"] } + # create user node if not existed + # user_ = self.db.get_or_create_user_v2(user) + + # _slide = None + user_embedding = "" + slide_document_embedding = "" + slide_weighted_avg_embedding_of_concepts = "" + + if recommendation_type in [ RecommendationType.PKG_BASED_KEYPHRASE_VARIANT, RecommendationType.PKG_BASED_DOCUMENT_VARIANT ]: + clu = rrh.save_and_get_concepts_modified( db=self.db, + rec_params=rec_params, + top_n=5, + user_embedding_status=True, + understood_list=body["understood_concept_ids"], + non_understood_list=body["non_understood_concept_ids"] + ) + + rec_params["concepts"] = clu["concepts"] + user_embedding = clu.get("user_embedding") + + elif recommendation_type in [ RecommendationType.CONTENT_BASED_KEYPHRASE_VARIANT, RecommendationType.CONTENT_BASED_DOCUMENT_VARIANT ]: + slide_concepts_ = self.db.get_main_concepts_by_slide_id(slide_id=body["slide_id"]) + _slide = self.db.get_slide(body["slide_id"]) + # slide_concepts = _slide[0]["s"]["concepts"] + slide_document_embedding = _slide[0]["s"]["initial_embedding"] + slide_weighted_avg_embedding_of_concepts = _slide[0]["s"][ + "weighted_embedding_of_concept" + ] + + rec_params["concepts"] = slide_concepts_ + rec_params["mid"] = body["material_id"] + clu = rrh.save_and_get_concepts_modified( db=self.db, + rec_params=rec_params, + top_n=len(slide_concepts_), + user_embedding_status=False, + understood_list=[], + non_understood_list=[] + ) + rec_params["concepts"] = clu["concepts"] + + factor_weights = rrh.build_factor_weights(body["rec_params"]["factor_weights"]) + result = self.process_recommandation_pipeline( + rec_params=rec_params, + factor_weights=factor_weights, recommendation_type=recommendation_type, + user_embedding=user_embedding, + slide_weighted_avg_embedding_of_concepts=slide_weighted_avg_embedding_of_concepts, + slide_document_embedding=slide_document_embedding, + top_n_resources=10 ) - result_video_ids = [] - result_article_ids = [] - - if not isinstance(youtube_videos, list) and not youtube_videos.empty: - result_video_ids = youtube_videos["id"].tolist() - - if not isinstance(wikipedia_articles, list) and not wikipedia_articles.empty: - result_article_ids = wikipedia_articles["id"].tolist() - - result_ids = result_video_ids + result_article_ids - - # filter the results from the database and keep only the ones generated by the Recommender - filtered_resources = [r for r in resources if r["rid"] in result_ids] - - # if not isinstance(youtube_videos, list) and (not youtube_videos.empty): - # youtube_videos = youtube_videos.drop(columns=["concept_embedding", "text", 'thumbnails', 'description', - # 'description_full', 'id', 'title', 'channelTitle', 'liveBroadcastContent', 'kind', 'publishedAt', - # 'channelId', 'publishTime', 'views', 'duration']) - # # file_path_youtube = "recommendation/data/video_concept_user_description_and_title_only.csv" - # file_path_youtube = "recommendation/data/video_concept_subtitles_only_15.csv" - # # file_path_youtube = "recommendation/data/video_document_subtitles_only_15.csv" - # # file_path_youtube = "recommendation/data/video_document_user.csv" - # # file_path_youtube = "recommendation/data/video_document_user_20.csv" - # # file_path_youtube = "recommendation/data/video_document_user_15.csv" - # # file_path_youtube = "recommendation/data/video_document_user_10.csv" - # # file_path_youtube = "recommendation/data/video_concept_user_description_and_title_only_15.csv" - # # file_path_youtube = "recommendation/data/video_concept_user_subtitles_only_15.csv" - # # file_path_youtube = "recommendation/data/video_concept_user_description_and_title_only_10.csv" - # - # # youtube_videos.to_csv(file_path_youtube, index=False) - # youtube_videos.to_csv(file_path_youtube, mode='a', index=False, header=False) - # - # if not isinstance(wikipedia_articles, list) and (not wikipedia_articles.empty): - # wikipedia_articles = wikipedia_articles.drop(columns=["concept_embedding", "text", 'abstract', 'id', - # 'title']) - # # file_path_wikipedia = "recommendation/data/article_concept_user_abstract_and_title_only.csv" - # file_path_wikipedia = "recommendation/data/article_concept_content_only_15.csv" - # # file_path_wikipedia = "recommendation/data/article_document_content_only_15.csv" - # # file_path_wikipedia = "recommendation/data/article_document_user.csv" - # # file_path_wikipedia = "recommendation/data/article_document_user_20.csv" - # # file_path_wikipedia = "recommendation/data/article_document_user_15.csv" - # # file_path_wikipedia = "recommendation/data/article_document_user_10.csv" - # # file_path_wikipedia = "recommendation/data/article_concept_user_abstract_and_title_only_15.csv" - # # file_path_wikipedia = "recommendation/data/article_concept_user_content_only_15.csv" - # # file_path_wikipedia = "recommendation/data/article_concept_user_abstract_and_title_only_10.csv" - # - # # wikipedia_articles.to_csv(file_path_wikipedia, index=False) - # wikipedia_articles.to_csv(file_path_wikipedia, mode='a', index=False, header=False) - - # file_path_youtube = "recommendation/dat2/video_data_all_without_user.png" - # file_path_wikipedia = "recommendation/dat2/articles_data_all_without_user.png" - # file_path_youtube_document_based_similarity = "recommendation/data/dist2/video_document_based_similarity.png" - # file_path_wikipedia_document_based_similarity = "recommendation/data/dist2/articles_document_based_similarity.png" - # file_path_youtube_concept_based_similarity = "recommendation/data/dist2/video_concept_based_similarity.png" - # file_path_wikipedia_concept_based_similarity = "recommendation/data/dist2/articles_concept_based_similarity.png" - # file_path_youtube_user_document_based_similarity = "recommendation/data/dist2/video_user_document_based_similarity.png" - # file_path_wikipedia_user_document_based_similarity = "recommendation/data/dist2/articles_user_document_based_similarity.png" - # file_path_youtube_user_concept_based_similarity = "recommendation/data/dist2/video_user_concept_based_similarity.png" - # file_path_wikipedia_user_concept_based_similarity = "recommendation/data/dist2/articles_user_concept_based_similarity.png" - # file_path_youtube_fused_similarity = "recommendation/data/dist2/video_fused_similarity.png" - # file_path_wikipedia_fused_similarity = "recommendation/data/dist2/articles_fused_similarity.png" - # file_path_youtube_fused_user_similarity = "recommendation/data/dist2/video_fused_user_similarity.png" - # file_path_wikipedia_fused_user_similarity = "recommendation/data/dist2/articles_fused_user_similarity.png" - # - # save_plot_data(youtube_videos, file_path_youtube) - # save_plot_data(wikipedia_articles, file_path_wikipedia) - # - # - # plot_distribution_chart(youtube_videos, "document_based_similarity", file_path_youtube_document_based_similarity) - # plot_distribution_chart(wikipedia_articles, "document_based_similarity", file_path_wikipedia_document_based_similarity) - # plot_distribution_chart(youtube_videos, "concept_based_similarity", file_path_youtube_concept_based_similarity) - # plot_distribution_chart(wikipedia_articles, "concept_based_similarity", file_path_wikipedia_concept_based_similarity) - # plot_distribution_chart(youtube_videos, "user_document_based_similarity", file_path_youtube_user_document_based_similarity) - # plot_distribution_chart(wikipedia_articles, "user_document_based_similarity", file_path_wikipedia_user_document_based_similarity) - # plot_distribution_chart(youtube_videos, "user_concept_based_similarity", file_path_youtube_user_concept_based_similarity) - # plot_distribution_chart(wikipedia_articles, "user_concept_based_similarity", file_path_wikipedia_user_concept_based_similarity) - # plot_distribution_chart(youtube_videos, "fused_similarity", file_path_youtube_fused_similarity) - # plot_distribution_chart(wikipedia_articles, "fused_similarity", file_path_wikipedia_fused_similarity) - # plot_distribution_chart(youtube_videos, "fused_user_similarity", file_path_youtube_fused_user_similarity) - # plot_distribution_chart(wikipedia_articles, "fused_user_similarity", file_path_wikipedia_fused_user_similarity) - - resp = get_serialized_resource_data(filtered_resources, concepts, relations=[]) - return resp - - -def get_serialized_resource_data(resources, concepts, relations): - """ """ - data = {} - ser_resources = [] - ser_realations = [] - - for resource in resources: - r = { - "title": resource["title"], - "id": resource["rid"], - "uri": resource["uri"], - "helpful_counter": resource["helpful_count"], - "not_helpful_counter": resource["not_helpful_count"], - "labels": resource["labels"], - "similarity_score": resource["similarity_score"], - "keyphrases": resource["keyphrases"], - } - - if "Video" in r["labels"]: - r["description"] = resource["description"] - r["description_full"] = resource["description_full"] - r["thumbnail"] = resource["thumbnail"] - r["duration"] = resource["duration"] - r["views"] = resource["views"] - r["publish_time"] = resource["publish_time"] - - elif "Article" in r["labels"]: - r["abstract"] = resource["abstract"] - - ser_resources.append({"data": r}) - for relation in relations: - r = { - "type": relation["type"], - "source": relation["source"], - "target": relation["target"], - } - ser_realations.append({"data": r}) - data["concepts"] = concepts - data["nodes"] = ser_resources - data["edges"] = ser_realations - - return data - - -def save_data_to_file(data, file_path): - data.to_csv(file_path, sep="\t", encoding="utf-8") + return result + + + def _get_resources_by_main_concepts(self, mid: str): #, slide_id: str): + # list of main concepts + main_concepts = self.db.get_main_concepts_by_mid(mid=mid) + try: + self.resources_crawler_logic(concepts=main_concepts) + return {"msg": True} + except Exception as e: + print(e) + pass + return {"msg": False} diff --git a/coursemapper-kg/recommendation/app/services/course_materials/recommendation/resource_recommender_helper.py b/coursemapper-kg/recommendation/app/services/course_materials/recommendation/resource_recommender_helper.py new file mode 100644 index 000000000..ca4026905 --- /dev/null +++ b/coursemapper-kg/recommendation/app/services/course_materials/recommendation/resource_recommender_helper.py @@ -0,0 +1,574 @@ +import json +from datetime import datetime, timedelta +from dateutil.parser import parse as date_parse +import numpy as np +from sklearn.preprocessing import normalize as normalize_sklearn, MinMaxScaler as MinMaxScaler_sklearn +import json +import threading +import time +import math +import scipy.stats as st +from ..db.neo4_db import NeoDataBase +from .recommendation_type import RecommendationType +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, Future + + +from log import LOG +import logging +logger = LOG(name=__name__, level=logging.DEBUG) + + +def insert_dict_if_not_exists_by_id(lst: list, new_dict: dict, key: str): + if not any(d.get(key) == new_dict.get(key) for d in lst): + lst.append(new_dict) + return lst + +def remove_duplicates_from_resources(dict_list: list, key: str="rid"): + ''' + Remove Duplicates from list (resource list) + ''' + logger.info("Remove Duplicates from list (resource list)") + + seen = set() + unique_dicts = [] + + for d in dict_list: + value = d[key] + if value not in seen: + seen.add(value) + unique_dicts.append(d) + + return unique_dicts + +def remove_keys_from_resources(resources: list, recommendation_type=None): + ''' + Remove keys from list (resource list) + keys = ["keyphrase_counts", "keyphrases", "keyphrases_infos", "keyphrase_embedding", "document_embedding"] + ''' + logger.info("Remove keys from list (resource list)") + if recommendation_type in [ RecommendationType.PKG_BASED_KEYPHRASE_VARIANT, RecommendationType.CONTENT_BASED_KEYPHRASE_VARIANT ]: + keys = [ "keyphrase_counts", "keyphrases_infos", "keyphrase_embedding", "document_embedding", + "composite_score", "labels", "concept_cid" + ] + else: + keys = [ "keyphrase_counts", "keyphrases_infos", "keyphrase_embedding", "document_embedding", + "composite_score", "labels", "concept_cid", "keyphrases" + ] + + resources_updated = [{k: v for k, v in d.items() if k not in keys} for d in resources] + return resources_updated + +def check_keys_not_empty_from_resources(resources: list, recommendation_type_str: str): + ''' + Check whether some resource attributes are empty or not, + such as: keyphrases, keyphrase_embedding, document_embedding + return: False (not empty) | True (empty) + ''' + logger.info("Check whether some resource attributes are empty or not, such as: keyphrases, keyphrase_embedding, document_embedding") + + for resource in resources: + if recommendation_type_str in ["1", "3"]: # keyphrase_embedding + if resource.get("keyphrase_embedding") == "": + return True + elif recommendation_type_str in ["2", "4"]: # document_embedding + if resource.get("document_embedding") == "": + return True + return False + +def get_top_n_concepts(concepts: list, top_n=5): + ''' + Get top n concepts from the list + ''' + # concepts.sort(key=lambda x: x["weight"], reverse=True) + concepts_ = sorted(concepts, key=lambda x: x["weight"]) + return concepts_[:top_n] + +def save_and_get_concepts_modified(db: NeoDataBase, rec_params, top_n=5, user_embedding_status=False, understood_list=[], non_understood_list =[]): + ''' + rec_params : {'user_id': str, 'mid': str, 'slide_id': int, 'category': str, 'concepts': list, 'recommendation_type': str, 'factor_weights': dict} + status: dnu or u (PKG_BASED_KEYPHRASE_VARIANT | PKG_BASED_DOCUMENT_VARIANT) | + content (CONTENT_BASED_DOCUMENT_VARIANT | CONTENT_BASED_KEYPHRASE_VARIANT) + u: for type 1, 2 (understood) + dnu_reset: not used for pkg recommandations + ''' + logger.info("Saving Logic: Concept_modified") + + result = { + "concepts": [], + "concept_cids": [], + "concept_names": [], + "rec_params": None, + "user_embedding": None + } + concepts_modified = [] + user_id = rec_params["user_id"] + + concepts: list = rec_params["concepts"] + cids = [concept["cid"] for concept in concepts] + # concepts.sort(key=lambda x: x["weight"], reverse=True) + # concepts = concepts[:top_n] + + # update status between understood and non-understood + if len(understood_list) > 0: + for cid in understood_list: + db.update_rs_btw_user_and_cm(user_id=user_id, cid=cid, weight=None, mid=None, status='u', only_status=True) + + if len(concepts) > 0: + if rec_params["recommendation_type"] in ["1", "2"]: + status = 'dnu' + else: + status = 'content' # for CONTENT_BASED_ + + for concept in concepts: + db.create_concept_modified(cid=concept["cid"]) + concept_modified = db.update_rs_btw_user_and_cm(user_id=user_id, cid=concept["cid"], weight=concept["weight"], mid=rec_params["mid"], status=status) + concepts_modified.append(concept_modified) + + if status == "dnu": + db.update_rs_btw_user_and_cms(user_id=user_id, cids=cids, special_status="dnu_reset") + + + # update user embedding value (because weight value could be changed from the user) + if user_embedding_status: + user_embedding = db.get_user_embedding_with_concept_modified(user_id=user_id, mid=rec_params["mid"], status=status) + result["user_embedding"] = user_embedding + + result["concepts"] = concepts + return result + +def normalize_factor_weights(factor_weights: dict=None, values: list=[], method_type = "l1", complete=True, sum_value=True): # List[float] + ''' + method_type: normalization techniques + l1: L1 normalization, also known as L1 norm normalization or Manhattan normalization + l1: L2 normalization, also known as L2 norm normalization or Euclidean normalization + max: Max Normalization + min-max: Min-Max + + https://www.pythonprog.com/sklearn-preprocessing-normalize/#Normalization_Techniques + TypeScript: https://sklearn.vercel.app/guide/install + + factor_weights : {'similarity_score': 0.7, 'creation_date': 0.3, 'views': 0.3, 'like_count': 0.1, 'user_rating': 0.1, 'saves_count': 0.1} + ''' + logger.info("Normalization of factor weights") + if len(factor_weights) == 0: + return {} + + normalized_values = None + scaled_data = None + + if factor_weights: + values = [value for key, value in factor_weights.items()] + key_names = [key for key, value in factor_weights.items()] + + if method_type == "l1": + normalized_values = normalize_sklearn([values], norm=method_type).tolist() + if method_type == "l2": + normalized_values = normalize_sklearn([values], norm=method_type).tolist() + if method_type == "max": + normalized_values = normalize_sklearn([values], norm=method_type).tolist() + if method_type == "min-max": + data = np.array(values).reshape(-1, 1) + scaler = MinMaxScaler_sklearn() + scaler.fit(data) + scaled_data = scaler.transform(data) + scaled_data = scaled_data.tolist() + scaled_data = [value[0] for value in scaled_data] + + if normalized_values: + normalized_values = normalized_values[0] + normalized_values = [round(value, 3) for value in normalized_values] + elif scaled_data: + normalized_values = scaled_data + + if sum_value: + # print("sun values: ", sum(normalized_values)) + logger.info("factor weight sum ->", sum(normalized_values)) + + if complete: + normalized_values = dict(zip(key_names, normalized_values)) + + return normalized_values + +def wilson_lower_bound_score(up, down, confidence=0.95): + """ + Calculate lower bound of wilson score + :param up: No of positive ratings + :param down: No of negative ratings + :param confidence: Confidence interval, by default is 95 % + :return: Wilson Lower bound score + """ + n = up + down + if n == 0: + return 0.0 + z = st.norm.ppf(1 - (1 - confidence) / 2) + phat = 1.0 * up / n + return (phat + z * z / (2 * n) - z * math.sqrt((phat * (1 - phat) + z * z / (4 * n)) / n)) / (1 + z * z / n) + +def normalize_min_max_score_date(date_str: str, max: datetime): + """ + Calculate Normalization Score of Creation Date + """ + date = date_parse(date_str).replace(tzinfo=None) + + # First video posted on Youtube + min = datetime(year=2005, month=4, day=23, hour=8, minute=31, second=52, tzinfo=None) + return (date - min).days / (max - min).days + +def normalize_min_max_score(value: int, min_value: int, max_value: int): + if (max_value - min_value) == 0: + return 0 + return (value - min_value) / (max_value - min_value) + +def calculate_factors_weights(category: int, resources: list, weights: dict = None, light=False): + """ + Sort by these extra features provided by the resources such as: + weights: dict containing factors weight + {'similarity_score': 0.2, 'creation_date': 0.2, 'views': 0.3, 'like_count': 0.1, 'user_rating': 0.1, 'saves_count': 0.1} + """ + now = datetime.now() + default_weight = 0.001 + + weight_similarity_score = weights.get("similarity_score") if 'similarity_score' in weights else default_weight + weight_creation_date = weights.get("creation_date") if 'creation_date' in weights else default_weight + weight_views = weights.get("views") if 'views' in weights else default_weight + weight_user_rating = weights.get("user_rating") if 'user_rating' in weights else default_weight + weight_like_count = weights.get("like_count") if 'like_count' in weights else default_weight + weight_saves_count = weights.get("saves_count") if 'saves_count' in weights else default_weight + + bookmarked_min_value = min(resources, key=lambda x: x["bookmarked_count"])["bookmarked_count"] + bookmarked_max_value = max(resources, key=lambda x: x["bookmarked_count"])["bookmarked_count"] + + similarity_score_min_value = min(resources, key=lambda x: x["similarity_score"])["similarity_score"] + similarity_score_max_value = max(resources, key=lambda x: x["similarity_score"])["similarity_score"] + + if category == 1: + min_views = int(min(resources, key=lambda x: int(x["views"]))["views"]) + max_views = int(max(resources, key=lambda x: int(x["views"]))["views"]) + + like_count_min_value = min(resources, key=lambda x: x["like_count"])["like_count"] + like_count_max_value = max(resources, key=lambda x: x["like_count"])["like_count"] + + for resource in resources: + similarity_normalized = normalize_min_max_score(value=int(resource["similarity_score"]), min_value=similarity_score_min_value, max_value=similarity_score_max_value) # resource["similarity_score"] + rating_normalized = wilson_lower_bound_score(up=resource["helpful_count"], down=resource["not_helpful_count"]) + creation_date_normalized = normalize_min_max_score_date(date_str=resource["publish_time"], max=now) + views_normalzed = normalize_min_max_score(value=int(resource["views"]), min_value=min_views, max_value=max_views) + bookmarked_normalized = normalize_min_max_score(value=int(resource["bookmarked_count"]), min_value=bookmarked_min_value, max_value=bookmarked_max_value) + like_count_normalized = normalize_min_max_score(value=int(resource["like_count"]), min_value=like_count_min_value, max_value=like_count_max_value) + + resource["composite_score"] = (views_normalzed * weight_views) \ + + (rating_normalized * weight_user_rating) \ + + (creation_date_normalized * weight_creation_date) \ + + (similarity_normalized * weight_similarity_score) \ + + (bookmarked_normalized * weight_saves_count) \ + + (like_count_normalized * weight_like_count) + + elif category == 2: + for resource in resources: + rating_normalized = wilson_lower_bound_score(up=resource["helpful_count"], down=resource["not_helpful_count"]) + bookmarked_normalized = normalize_min_max_score(value=int(resource["bookmarked_count"]), min_value=bookmarked_min_value, max_value=bookmarked_max_value) + similarity_normalized = normalize_min_max_score(value=int(resource["similarity_score"]), min_value=similarity_score_min_value, max_value=similarity_score_max_value) + + resource["composite_score"] = (rating_normalized * weight_user_rating) \ + + (bookmarked_normalized * weight_saves_count) \ + + (similarity_normalized * weight_similarity_score) + # + (resource["bookmarked_count"] * weight_saves_count) \ + # + (resource["similarity_score"] * weight_similarity_score) + + + # sort by composite score value + resources.sort(key=lambda x: x["composite_score"], reverse=True) + + return resources + +def rank_resources_proportional_top_n_with_remainder_by_concept_cid(dict_list, top_n=10): + # Group dictionaries by their 'concept_cid' + grouped_by_id = defaultdict(list) + for item in dict_list: + grouped_by_id[item['concept_cid']].append(item) + + # Calculate the proportional selection count for each group + total_items = sum(len(group) for group in grouped_by_id.values()) + proportions = {k: math.ceil((len(v) / total_items) * top_n) for k, v in grouped_by_id.items()} + + # Adjust the proportions so their sum equals `top_n` + while sum(proportions.values()) > top_n: + for key in proportions: + if proportions[key] > 0: + proportions[key] -= 1 + if sum(proportions.values()) == top_n: + break + + + # Collect top items based on calculated proportions and the remainder + top_items = [] + remainder_items = [] + + for key, group in grouped_by_id.items(): + # Sort the group if necessary and separate top and remaining items + top_count = proportions[key] + top_items.extend(group[:top_count]) + remainder_items.extend(group[top_count:]) + + # Combine top 10 items with the remainder items + final_result = top_items + remainder_items + final_result = remove_duplicates_from_resources(final_result) + return final_result + + +def rank_resources(resources: list, weights: dict = None, recommendation_type=None, pagination_params: dict = None, top_n_resources=10): + ''' + factor_weights: { "video": dict, (default: {'like_count': 0.146, 'creation_date': 0.205, 'views': 0.146, 'similarity_score': 0.152, 'saves_count': 0.199, 'user_rating': 0.152}) + "article": dict, (default: {'similarity_score': 0.3, 'saves_count': 0.4, 'user_rating': 0.3}) + } + Step 1: Remove duplicates if exist + Step 2: Ranking/Sorting Logic for Resources + Result Form: {"articles": list, "videos": list} + Step 3: Last Step: Resources having Rating related to DNU_modified (cid) + ''' + logger.info("Appliying Ranking Algorithm") + resources_articles = [] + resources_videos = [] + + if len(resources) > 0: + video_weights_normalized = normalize_factor_weights(factor_weights=weights["video"], method_type="l1", complete=True, sum_value=False) + article_weights_normalized = normalize_factor_weights(factor_weights=weights["article"], method_type="l1", complete=True, sum_value=False) + + # video items + resources_videos = [resource for resource in resources if "Video" in resource["labels"]] + if len(video_weights_normalized) > 0: + resources_videos = calculate_factors_weights(category=1, resources=resources_videos, weights=video_weights_normalized) + + resources_videos = rank_resources_proportional_top_n_with_remainder_by_concept_cid(resources_videos) + resources_videos = remove_keys_from_resources(resources=resources_videos, recommendation_type=recommendation_type) + + # articles items + resources_articles = [resource for resource in resources if "Article" in resource["labels"]] + if len(article_weights_normalized) > 0: + resources_articles = calculate_factors_weights(category=2, resources=resources_articles, weights=article_weights_normalized) + + resources_articles = rank_resources_proportional_top_n_with_remainder_by_concept_cid(resources_articles) + resources_articles = remove_keys_from_resources(resources=resources_articles, recommendation_type=recommendation_type) + + return { + "articles": get_paginated_resources(resources_articles, pagination_params), # resources_articles[: top_n_resources], + "videos": get_paginated_resources(resources_videos, pagination_params) # resources_videos[: top_n_resources] + } + # return { "articles": [], "videos": [] } + +def get_paginated_resources(resources: list, pagination_params: dict=None): + ''' + Simulate Pagination Logic with Resource List + pagination_params: { + "page_number": 1, + "page_size": 10 + } + ''' + page_number = 1 + page_size = 10 + if pagination_params: + page_number = pagination_params["page_number"] + page_size = pagination_params["page_size"] + + total_items = len(resources) + total_pages = -(-total_items // page_size) + start_index = (page_number - 1) * page_size + end_index = min(start_index + page_size, total_items) + + return { + "current_page": page_number, + "total_pages": total_pages, + "total_items": total_items, + "content": resources[start_index:end_index] + } + + +def rec_params_request_mapped(data_rec_params: dict, data_default: dict=None): + ''' + Map Recommandations Params + ''' + + understood = data_default.get("understoodConcepts") + non_understood = data_default.get("nonUnderstoodConcepts") + new_concepts = data_default.get("newConcepts") + + return { + "material_id": data_default.get("materialId"), + "slide_id": data_default.get("slideId"), + "understood": understood, + "non_understood": non_understood, + "new_concepts": new_concepts, + "understood_concept_ids": [cid for cid in understood.split(",") if understood], + "non_understood_concept_ids": [ cid for cid in non_understood.split(",") if non_understood ], + "new_concept_ids": [cid for cid in new_concepts.split(",") if new_concepts], + "username": data_default.get("username"), + "user_id": data_default.get("userId"), + "user_email": data_default.get("userEmail"), + "rec_params": data_rec_params, + } + +def build_factor_weights(factor_weights_params: dict = None): + ''' + Generate Factor weights for Ranking + ''' + factor_weights_videos = {} + factor_weights_articles = {} + + if factor_weights_params: + factor_weights_videos = normalize_factor_weights( factor_weights=factor_weights_params, + method_type="l1", + complete=True, + sum_value=False + ) + # set article weights + for key, value in factor_weights_params.items(): + if key in ["similarity_score", "user_rating", "saves_count"]: + factor_weights_articles[key] = value + + factor_weights_articles = normalize_factor_weights( factor_weights=factor_weights_articles, + method_type="l1", + complete=True, + sum_value=False + ) + return { + "video": factor_weights_videos, + "article": factor_weights_articles + } + +def check_request_temp(rec_params: dict, key="cid"): + ''' + check if user add or change concpet(s) to the concepts list + check if resources (saved temporally: Redis) have already been recommended for the concepts given + + key = rec_params_user_id + get temp result_temp : {concepts: list, resources: list} + get temporal rec_params_concepts + if there are same from those stored in the temp + and return resources temp stored + ''' + are_concepts_sane = True + resources = None + user_id = rec_params["user_id"] + concepts = rec_params["concepts"] + + # result_temp = get_redis_key_value(key_name=f"{user_id}_{redis_key_1}") + result_temp = None + if result_temp: + result_temp = json.loads(result_temp) + concepts_temp = result_temp["concepts"] + + if concepts_temp is None: + are_concepts_sane = False + + elif len(concepts) != len(concepts_temp): + are_concepts_sane = False + + elif len(concepts) == len(concepts_temp): + for dict1, dict2 in zip(concepts, concepts_temp): + if dict1.get(key) != dict2.get(key): + are_concepts_sane = False + break + + resources = result_temp["resources"] + else: + are_concepts_sane = False + return are_concepts_sane, resources + +def create_get_resources_thread(func, args): + ''' + active threading for the function given + func: the function to run + args: arguments taken from the function func + ''' + thread = threading.Thread(target=func, args=args) + thread.start() + # redis_client.set(name=f"{user_id}_{redis_key_1}", value=status, ex=(redis_client_expiration_time * 10080)) + +def check_and_validate_resources(db: NeoDataBase, concepts: list ): + ''' + Check if concepts already exist and connected to any resources in Neo4j Database + If resources exist, check based on: + updated_at: it's not more than a given time (one week old) + Video: default time = 30 days + Article: default time = 365 days + ''' + concepts_db_checked = [] + for concept in concepts: + cid = concept["cid"] + concept_updated = { "cid": cid, "name": concept["name"], "weight": concept["weight"], + "is_video_too_old": False, "is_article_too_old": False, + "resources_video_exist": False, "resources_article_exist": False + } + + # content type: Video + count_resourses_videos_only_exist = db.retrieve_resources_by_updated_at_exist_or_counts(cids=[cid], content_type="Video", only_exist=True) + count_resourses_videos = db.retrieve_resources_by_updated_at_exist_or_counts(cids=[cid], content_type="Video", only_exist=False, days=30) + + # content type: Article + count_resourses_articles_only_exist = db.retrieve_resources_by_updated_at_exist_or_counts(cids=[cid], content_type="Article", only_exist=True) + count_resourses_articles = db.retrieve_resources_by_updated_at_exist_or_counts(cids=[cid], content_type="Article", only_exist=False, days=365) + + if count_resourses_videos < 1: + concept_updated["is_video_too_old"] = True + if count_resourses_articles < 1: + concept_updated["is_article_too_old"] = True + + if count_resourses_videos_only_exist > 0: + concept_updated["resources_video_exist"] = True + if count_resourses_articles_only_exist > 0: + concept_updated["resources_article_exist"] = True + + concepts_db_checked.append(concept_updated) + return concepts_db_checked + +def parallel_crawling_resources(function, concept_updated, result_type: str, top_n_videos, top_n_articles): + ''' + Parallel Crawling of Resources with the function + canditate_selection from Class Recommender + submit function: takes + function: canditate_selection (this function takes the params below): it's the main function that send query to YouTube and Wikipedia APIs + query, video, result_type="records", top_n_videos=2, top_n_articles=2 + concept_updated: dict{} | cid, + ''' + with ThreadPoolExecutor() as executor: + future_videos = Future().set_result([]) + if concept_updated["is_video_too_old"] == True: + future_videos = executor.submit(function, concept_updated["name"], True, result_type, top_n_videos, top_n_articles) + if concept_updated["resources_video_exist"] == False: + future_videos = executor.submit(function, concept_updated["name"], True, result_type, top_n_videos, top_n_articles) + + future_articles = Future().set_result([]) + if concept_updated["is_article_too_old"] == True: + future_articles = executor.submit(function, concept_updated["name"], False, result_type, top_n_videos, top_n_articles) + if concept_updated["resources_article_exist"] == False: + future_articles = executor.submit(function, concept_updated["name"], False, result_type, top_n_videos, top_n_articles) + + result_videos = future_videos.result() if future_videos is not None else [] + result_articles = future_articles.result() if future_articles is not None else [] + + return { "cid": concept_updated["cid"], + "videos": result_videos, + "articles": result_articles, + "resources_video_exist": concept_updated["resources_video_exist"], + "resources_article_exist": concept_updated["resources_article_exist"], + "is_video_too_old": concept_updated["is_video_too_old"], + "is_article_too_old": concept_updated["is_video_too_old"], + } + + +def parallel_crawling_resources_v2(function, concept_name: str, cid: str, result_type: str, top_n_videos, top_n_articles): + ''' + Parallel Crawling of Resources with the function + canditate_selection from Class Recommender + submit function: takes + function: canditate_selection (this function takes the params below) + query, video, result_type="records", top_n_videos=2, top_n_articles=2 + ''' + with ThreadPoolExecutor() as executor: + future_videos = executor.submit(function, concept_name, True, result_type, top_n_videos, top_n_articles) + future_articles = executor.submit(function, concept_name, False, result_type, top_n_videos, top_n_articles) + result_videos = future_videos.result() + result_articles = future_articles.result() + return {"cid": cid, "videos": result_videos, "articles": result_articles} + diff --git a/coursemapper-kg/recommendation/app/services/course_materials/recommendation/wikipedia_service.py b/coursemapper-kg/recommendation/app/services/course_materials/recommendation/wikipedia_service.py index fce8c3bec..cc167f190 100644 --- a/coursemapper-kg/recommendation/app/services/course_materials/recommendation/wikipedia_service.py +++ b/coursemapper-kg/recommendation/app/services/course_materials/recommendation/wikipedia_service.py @@ -16,13 +16,14 @@ def get_articles(self, concepts, top_n=15): logger.info("Get Wikipedia Articles") response = wikipedia.search(concepts, top_n) + data = [] w_data = [] for title in response: try: - page = wikipedia.page(title) - content = page.content + page = wikipedia.page(title) # auto_suggest=False + # content = page.content abstract = page.summary url = page.url # thumbnail_url = page.images[0] if len(page.images) > 0 else "" @@ -34,7 +35,8 @@ def get_articles(self, concepts, top_n=15): # print("w_data", data) except (PageError, DisambiguationError) as e: - logger.error("title {} raised a PageError".format(title), e) + # logger.error("title {} raised a PageError".format(title), e) + pass finally: # w_data = pd.DataFrame(data, columns=["id", "text", "abstract", "title", "thumbnail_url"]) w_data = pd.DataFrame( diff --git a/coursemapper-kg/recommendation/app/services/course_materials/recommendation/youtube_service.py b/coursemapper-kg/recommendation/app/services/course_materials/recommendation/youtube_service.py index 77c9822c1..beefc5cba 100644 --- a/coursemapper-kg/recommendation/app/services/course_materials/recommendation/youtube_service.py +++ b/coursemapper-kg/recommendation/app/services/course_materials/recommendation/youtube_service.py @@ -12,6 +12,7 @@ import logging from log import LOG import time +from googleapiclient.errors import HttpError logger = LOG(name=__name__, level=logging.DEBUG) @@ -42,78 +43,131 @@ class YoutubeService: api_version, developerKey="AIzaSyClxnNwQ1x34pGioQazLlGxOjO9Fp2GGTY", ) - - def get_videos(self, concepts, top_n=15): - logger.info("Get Videos") - logger.info(concepts) - video_data = [] - duration_list = [] - view_list = [] - description_list = [] + DEVELOPER_KEYS = [ + "AIzaSyD_CGmR_Voq4DIV5okRaR6G8adoe-ZSZsM", + "AIzaSyClxnNwQ1x34pGioQazLlGxOjO9Fp2GGTY", + "AIzaSyADNntK6m7DbA6eZFYOa9Y8e6IYHykUUFE", + "AIzaSyBphZOn7EJmPMmZwrB71aepaA5Rbuex9MU", + "AIzaSyB2Wck31LUlgsqI7dgTcC2dMeeVXgb9TDI", + ] + + def search_youtube_videos(self, developer_keys, query, top_n=50, api_service_name="youtube", api_version="v3"): + """ + Switching YouTube API keys + """ retry_count = 3 retry_delay = 5 - - for i in range(retry_count): + i = 0 + for key in developer_keys: try: - # your code that makes the YouTube API request - request = self.youtube.search().list( + youtube = googleapiclient.discovery.build(api_service_name, api_version, developerKey=key) + request = youtube.search().list( part="snippet", maxResults=top_n, type="video", - q=concepts, + q=query, relevanceLanguage="en", ) + return request.execute(), youtube - response = request.execute() - break # exit the loop if the request succeeds except (ConnectionAbortedError, ConnectionResetError, timeout) as e: logger.error("Error while getting the videos") logger.error(e) if i == retry_count - 1: raise # re-raise the exception if all retries fail - delay = retry_delay * ( - 2**i - ) # use a backoff algorithm to increase the delay + delay = retry_delay * (2 ** i) # use a backoff algorithm to increase the delay time.sleep(delay) logger.info("New Try") + if retry_count == 0: + return None, None + except HttpError as e: + if e.resp.status == 403 and "quota" in str(e): + print(f"Quota exceeded for key: {key}. Trying next key...") + else: + raise e + raise Exception("All API keys have exceeded their quota.") + + def get_videos(self, concepts, top_n=15): + logger.info("Get Videos") + # logger.info(concepts) + video_data = [] + duration_list = [] + view_list = [] + description_list = [] + like_count_list = [] + channel_title_list = [] + retry_count = 3 + retry_delay = 5 + + # Switching keys + response, youtube_api_sinlge = self.search_youtube_videos( + developer_keys=self.DEVELOPER_KEYS, query=concepts, top_n=top_n + ) + if len(response["items"]) == 0: logger.info("No Video found for this input") return [] else: df_items = pd.DataFrame(response["items"]) + + # Build id frame and keep your rename to "id" df_ids_list = df_items["id"].to_list() df_ids = pd.DataFrame(df_ids_list) df_ids.rename(columns={"videoId": "id"}, inplace=True) + # Build snippet frame df_snippet_list = df_items["snippet"].to_list() df_snippet = pd.DataFrame(df_snippet_list) - # df_snippet["text"] = pd.DataFrame(df_snippet["title"] + ". " + df_snippet["description"]) - # df_snippet["text"] = df_snippet["text"].str.lower() - # df_snippet["text"] = pd.DataFrame(df_snippet["title"]) - # df_snippet["query"] = concepts + + # Avoid join collision if both have "channelId" + if "channelId" in df_ids.columns and "channelId" in df_snippet.columns: + df_ids = df_ids.drop(columns=["channelId"]) + for index, id in enumerate(df_ids["id"]): # try: - # df_snippet["text"][index] = df_snippet["text"][index] + ". " + get_subtitles(id) + # df_snippet["text"][index] = df_snippet["text"][index] + ". " + get_subtitles(id) # except (NoTranscriptFound, TranscriptsDisabled) as e: - # logger.error("No transcript found in english or transcript disabled for this video " - # "https://www.youtube.com/watch?v={} ".format(id)) + # logger.error("No transcript found in english or transcript disabled for this video " + # "https://www.youtube.com/watch?v={} ".format(id)) try: - duration, views, description = self.get_video_details(id) - duration = re.findall(r"\d+", duration) - duration = ":".join(duration) - # print(id, duration, views) - duration_list.append(duration) - view_list.append(views) - description_list.append(description) - except Exception as e: - logger.error("Error while getting the videos details", e) + res = self.get_video_details(youtube_api_sinlge, id) + if res is None: + raise ValueError("No details returned for video id {}".format(id)) + duration, views, description, like_count, channel_title = res + + # Keep your duration normalization + duration = re.findall(r"\d+", str(duration)) + duration = ":".join(duration) + except Exception: + # Fix logging formatter error and keep same message semantics + logger.exception("Error while getting the videos details for id %s", id) + # Append safe defaults to keep list lengths aligned with df_ids + duration = "0" + views = 0 + description = "" + like_count = 0 + channel_title = "" + + duration_list.append(duration) + view_list.append(views) + description_list.append(description) + like_count_list.append(like_count) + channel_title_list.append(channel_title) + + # Keep your join, now safe from column overlap video_data = df_ids.join(df_snippet) + + # Assign lists (same behavior as before, now lengths aligned) video_data["duration"] = pd.Series(duration_list) video_data["views"] = pd.Series(view_list) video_data["description_full"] = pd.Series(description_list) + video_data["like_count"] = pd.Series(like_count_list) + video_data["channel_title"] = pd.Series(channel_title_list) + + # Keep your text construction and casing video_data["text"] = pd.DataFrame( video_data["title"] + ". " + video_data["description"] ) @@ -123,50 +177,41 @@ def get_videos(self, concepts, top_n=15): return video_data - def get_video_details(self, video_id): - print("get_video_details for id -------------------- ", video_id) - r = ( - self.youtube.videos() - .list( - part="snippet,statistics,contentDetails", - id=video_id, - fields="items(statistics," + "contentDetails(duration),snippet)", - ) - .execute() - ) + def get_video_details(self, youtube_api_sinlge, video_id): + # print("get_video_details for id -------------------- ", video_id) try: - duration = ( - r["items"][0]["contentDetails"]["duration"] - if r["items"][0]["contentDetails"]["duration"] - else 0 - ) - views = ( - r["items"][0]["statistics"]["viewCount"] - if r["items"][0]["statistics"]["viewCount"] - else 0 - ) - description = ( - r["items"][0]["snippet"]["description"] - if r["items"][0]["snippet"]["description"] - else "" - ) - - except Exception as e: - print("---------------------------------------") - print(e) - # The number of views are not present for some videos and this leads to an exception. For this - # reason a default value of 0 views will be given that video. - views = 0 - duration = ( - r["items"][0]["contentDetails"]["duration"] - if r["items"][0]["contentDetails"]["duration"] - else 0 - ) - description = ( - r["items"][0]["snippet"]["description"] - if r["items"][0]["snippet"]["description"] - else "" + r = ( + # self.youtube.videos() + youtube_api_sinlge.videos() + .list( + part="snippet,statistics,contentDetails", + id=video_id, + fields="items(statistics,contentDetails(duration),snippet)", + ) + .execute() ) - - return duration, views, description - return duration, views, description + except HttpError: + logger.exception("YouTube API error for id %s", video_id) + return None + except Exception: + logger.exception("Unexpected error calling YouTube for id %s", video_id) + return None + + # Defensive: items can be empty; return None so caller can handle + items = r.get("items", []) + if not items: + return None + + item = items[0] + cd = item.get("contentDetails", {}) or {} + st = item.get("statistics", {}) or {} + sn = item.get("snippet", {}) or {} + + # Safe gets with defaults (keep same return semantics) + duration = cd.get("duration") if cd.get("duration") else 0 + views = st.get("viewCount") if st.get("viewCount") else 0 + description = sn.get("description") if sn.get("description") else "" + like_count = st.get("likeCount") if st.get("likeCount") else 0 + channel_title = sn.get("channelTitle") if sn.get("channelTitle") else "" + + return duration, views, description, like_count, channel_title diff --git a/coursemapper-kg/recommendation/app/views/course_materials.py b/coursemapper-kg/recommendation/app/views/course_materials.py index ea028396a..0c61a98f0 100644 --- a/coursemapper-kg/recommendation/app/views/course_materials.py +++ b/coursemapper-kg/recommendation/app/views/course_materials.py @@ -2,6 +2,7 @@ from log import LOG import time +import json from ..services.course_materials.recommendation.resource_recommender import ResourceRecommenderService from ..services.course_materials.recommendation.recommendation_type import RecommendationType @@ -12,57 +13,38 @@ def get_concepts(job): - material_id = job["materialId"] - user_id = job["userId"] - understood = job["understood"] - non_understood = job["nonUnderstood"] - new_concepts = job["newConcepts"] + data = job["data"] + # material_page = data("materialPage") + material_id = data["materialId"] + user_id = data["userId"] + understood = data["understoodConcepts"] + non_understood = data["nonUnderstoodConcepts"] + new_concepts = data["newConcepts"] + + # print("not-understood:", non_understood, flush=True) understood = [cid for cid in understood.split(",") if understood] non_understood = [cid for cid in non_understood.split(",") if non_understood] new_concepts = [cid for cid in new_concepts.split(",") if new_concepts] material_id = material_id.split("-")[0] + # slide_id = str(material_id) + "_slide_" + str(material_page) - print( - "material_id:", - material_id, - "user_id: ", - user_id, - "understood: ", - understood, - "nonUnderstood: ", - non_understood, - "new_concepts: ", - new_concepts, - flush=True, - ) start_time1 = time.time() start_time = time.time() data_service = RecService() end_time = time.time() print("Get RecService time: ", end_time - start_time, flush=True) - # use GCN to get final embedding of each node + start_time = time.time() data_service._extract_vector_relation(mid=material_id) + # logger.info("GCN") gcn = GCN() gcn.load_data() - - ### ======== - ### LightGCN Variant - # from ..services.course_materials.GCN.lightGCN import LightGCN - # lightGCN = LightGCN() - # lightGCN.load_data(variant=True) - ### ======== - ### LightGCN - # from ..services.course_materials.GCN.lightGCN import LightGCN - # lightGCN = LightGCN() - # lightGCN.load_data(variant=False) - ### ======== - end_time = time.time() print("use gcn Execution time: ", end_time - start_time, flush=True) start_time = time.time() + # user = {"name": username, "id": user_id, "user_email": user_email} data_service._construct_user( user_id=user_id, non_understood=non_understood, @@ -85,89 +67,19 @@ def get_concepts(job): ) end_time1 = time.time() print("Execution time: ", end_time1 - start_time1, flush=True) - #make_response.headers.add('Access-Control-Allow-Origin', '*') - return resp -def get_resources(job): - material_id = job["materialId"] - user_id = job["userId"] - slide_id = job["slideId"] - understood = job["understood"] - non_understood = job["nonUnderstood"] - new_concepts = job["newConcepts"] - - print("not-understood from paul:", non_understood, flush=True) - understood_concept_ids = [cid for cid in understood.split(",") if understood] - non_understood_concept_ids = [ - cid for cid in non_understood.split(",") if non_understood - ] - new_concept_ids = [cid for cid in new_concepts.split(",") if new_concepts] - - print( - "material_id:", - material_id, - "slide_id: ", - slide_id, - "user_id: ", - user_id, - "understood: ", - understood, - "nonUnderstood: ", - non_understood, - "new_concepts: ", - new_concepts, - flush=True, - ) +def get_resources_by_main_concepts(job): + data = job["data"] + data = json.loads(data) + mid = data["materialId"] resource_recommender_service = ResourceRecommenderService() - # TODO comment out or remove the next line if the recommendation_type is sent from the frontend - # Only first model is needed ==> no model_type will be sent from frontend (other models were added for evaluation task) - recommendation_type = "1" - - # Check if parameters exist. If one doesn't exist, return not found message - check_message = resource_recommender_service.check_parameters( - slide_id=slide_id, - material_id=material_id, - non_understood_concept_ids=non_understood_concept_ids, - understood_concept_ids=understood_concept_ids, - new_concept_ids=new_concept_ids, - recommendation_type=recommendation_type, - ) - if check_message != "": - return make_response(check_message, 404) - - # Map recommendation type to enum values - - if recommendation_type == "1": - recommendation_type = RecommendationType.DYNAMIC_KEYPHRASE_BASED - elif recommendation_type == "2": - recommendation_type = RecommendationType.DYNAMIC_DOCUMENT_BASED - elif recommendation_type == "3": - recommendation_type = RecommendationType.STATIC_KEYPHRASE_BASED - elif recommendation_type == "4": - recommendation_type = RecommendationType.STATIC_DOCUMENT_BASED - - # If personalized recommendtion, build user model - if ( - recommendation_type != RecommendationType.WITHOUT_EMBEDDING - and recommendation_type != RecommendationType.COMBINED_STATIC - and recommendation_type != RecommendationType.STATIC_KEYPHRASE_BASED - and recommendation_type != RecommendationType.STATIC_DOCUMENT_BASED - ): - resource_recommender_service._construct_user( - user_id=user_id, - non_understood=non_understood_concept_ids, - understood=understood_concept_ids, - new_concepts=new_concept_ids, - mid=material_id, - ) + result = resource_recommender_service._get_resources_by_main_concepts(mid=mid) + return result - resp = resource_recommender_service._get_resources( - user_id=user_id, - slide_id=slide_id, - material_id=material_id, - recommendation_type=recommendation_type, - ) - - return resp +def get_resources(job): + data = job["data"] + resource_recommender_service = ResourceRecommenderService() + result = resource_recommender_service._get_resources(data_default=data["default"], data_rec_params=data["rec_params"]) + return result diff --git a/coursemapper-kg/recommendation/app/worker.py b/coursemapper-kg/recommendation/app/worker.py index 14c28ad69..d206af6d3 100644 --- a/coursemapper-kg/recommendation/app/worker.py +++ b/coursemapper-kg/recommendation/app/worker.py @@ -7,7 +7,7 @@ import io from log import LOG -from app.views.course_materials import get_concepts, get_resources +from app.views import course_materials as recs from app.shared import redis, worker_id, set_current_job_id from config import Config @@ -106,9 +106,11 @@ def start_worker(pipelines): try: # Run the pipeline if pipeline == 'concept-recommendation': - result = get_concepts(job) + result = recs.get_concepts(job) elif pipeline == 'resource-recommendation': - result = get_resources(job) + result = recs.get_resources(job) + # elif pipeline == 'get_resources_by_main_concepts': + # result = recs.get_resources_by_main_concepts(job) else: raise ValueError(f'Unknown pipeline: {pipeline}') diff --git a/coursemapper-kg/recommendation/compose.yaml b/coursemapper-kg/recommendation/compose.yaml index 94bd43fed..8b420377d 100644 --- a/coursemapper-kg/recommendation/compose.yaml +++ b/coursemapper-kg/recommendation/compose.yaml @@ -11,7 +11,7 @@ services: REDIS_PORT: 6379 REDIS_DB: 0 REDIS_PASSWORD: - PIPELINES: concept-recommendation,resource-recommendation + PIPELINES: concept-recommendation,resource-recommendation,get_resources_by_main_concepts depends_on: neo4j: condition: service_healthy diff --git a/coursemapper-kg/recommendation/example.env b/coursemapper-kg/recommendation/example.env index b0bb943bd..2057eccdf 100644 --- a/coursemapper-kg/recommendation/example.env +++ b/coursemapper-kg/recommendation/example.env @@ -5,7 +5,7 @@ REDIS_HOST=localhost REDIS_PORT=6379 REDIS_DATABASE=0 REDIS_PASSWORD= -PIPELINES=concept-recommendation,resource-recommendation +PIPELINES=concept-recommendation,resource-recommendation,get_resources_by_main_concepts YOUTUBE_API_KEY=AIzaSyD_CGmR_Voq4DIV5okRaR6G8adoe-ZSZsM YOUTUBE_API_KEY_2=AIzaSyADNntK6m7DbA6eZFYOa9Y8e6IYHykUUFE YOUTUBE_API_KEY_3=AIzaSyClxnNwQ1x34pGioQazLlGxOjO9Fp2GGTY diff --git a/coursemapper-kg/recommendation/log.py b/coursemapper-kg/recommendation/log.py index d835736fa..24b947685 100644 --- a/coursemapper-kg/recommendation/log.py +++ b/coursemapper-kg/recommendation/log.py @@ -22,7 +22,7 @@ def __init__( self.addHandler(self.get_console_handler()) if log_file: self.addHandler(self.get_file_handler()) - self.addHandler(self.get_redis_handler()) + # self.addHandler(self.get_redis_handler()) self.propagate = False def get_console_handler(self): diff --git a/coursemapper-kg/wp-pg/Dockerfile b/coursemapper-kg/wp-pg/Dockerfile index 83b577bdb..d97cdc8b9 100644 --- a/coursemapper-kg/wp-pg/Dockerfile +++ b/coursemapper-kg/wp-pg/Dockerfile @@ -1,16 +1,13 @@ +# syntax=docker/dockerfile:1.15 FROM postgres:16 -# APT requires archive URLs -COPY debian/sources.list /etc/apt/sources.list - # Install dependencies -ENV RUNTIME_DEPS "ca-certificates rclone" +ENV RUNTIME_DEPS="ca-certificates rclone" RUN --mount=type=cache,sharing=private,target=/var/cache/apt \ --mount=type=cache,sharing=private,target=/var/lib/apt < /etc/apt/apt.conf.d/keep-cache DEBIAN_FRONTEND=noninteractive apt-get update -q && - apt-get --reinstall install debian-archive-keyring && apt-get install -qq --no-install-recommends -o=Dpkg::Use-Pty=0 $RUNTIME_DEPS EOF diff --git a/coursemapper-kg/wp-pg/debian/sources.list b/coursemapper-kg/wp-pg/debian/sources.list deleted file mode 100644 index 606fec29a..000000000 --- a/coursemapper-kg/wp-pg/debian/sources.list +++ /dev/null @@ -1,3 +0,0 @@ -deb http://archive.debian.org/debian/ stretch main contrib non-free -deb http://archive.debian.org/debian/ stretch-proposed-updates main contrib non-free -deb http://archive.debian.org/debian-security stretch/updates main contrib non-free diff --git a/proxy/Dockerfile b/proxy/Dockerfile index 528d9aeb0..67d07d905 100644 --- a/proxy/Dockerfile +++ b/proxy/Dockerfile @@ -1,5 +1,5 @@ -# syntax=docker/dockerfile:1.5 -FROM nginxinc/nginx-unprivileged:1.27.5-alpine +# syntax=docker/dockerfile:1.15 +FROM nginxinc/nginx-unprivileged:1.31.0-alpine USER root COPY --link ./nginx/conf.d/* /etc/nginx/conf.d/ diff --git a/webapp/Dockerfile b/webapp/Dockerfile index 736d5d5f3..f0698800c 100644 --- a/webapp/Dockerfile +++ b/webapp/Dockerfile @@ -1,11 +1,11 @@ -# syntax=docker/dockerfile:1.5 -FROM node:22.1-slim as build +# syntax=docker/dockerfile:1.15 +FROM node:24.0-slim as build WORKDIR /app -ENV PATH "$PATH:/app/node_modules/.bin" -ENV NODE_ENV production -ENV NPM_CONFIG_UPDATE_NOTIFIER false -ENV NG_PERSISTENT_BUILD_CACHE 1 +ENV PATH="$PATH:/app/node_modules/.bin" +ENV NODE_ENV=production +ENV NPM_CONFIG_UPDATE_NOTIFIER=false +ENV NG_PERSISTENT_BUILD_CACHE=1 COPY package*.json . RUN --mount=type=cache,target=/app/.npm,rw \ @@ -17,7 +17,7 @@ RUN --mount=type=cache,target=/app/node_modules/.cache,rw \ -FROM nginxinc/nginx-unprivileged:1.27.5-alpine +FROM nginxinc/nginx-unprivileged:1.31.0-alpine USER root WORKDIR /usr/share/nginx/html @@ -28,5 +28,5 @@ COPY --link --from=build --chown=101 /app/dist/webapp/ ./ COPY --link ./nginx/conf.d/* /etc/nginx/conf.d/ COPY --link --chmod=0755 ./nginx/docker-entrypoint.d/* /docker-entrypoint.d/ -ENV NGINX_ENTRYPOINT_QUIET_LOGS 1 +ENV NGINX_ENTRYPOINT_QUIET_LOGS=1 EXPOSE 4200 diff --git a/webapp/dev.Dockerfile b/webapp/dev.Dockerfile index afb236303..7eb3086db 100644 --- a/webapp/dev.Dockerfile +++ b/webapp/dev.Dockerfile @@ -1,9 +1,9 @@ -# syntax=docker/dockerfile:1.5 -FROM node:22-slim +# syntax=docker/dockerfile:1.15 +FROM node:24-slim WORKDIR /app -ENV PATH "$PATH:/app/node_modules/.bin" -ENV NODE_ENV development +ENV PATH="$PATH:/app/node_modules/.bin" +ENV NODE_ENV=development # No files are added, source directory is expected to be mounted diff --git a/webapp/src/app/components/footer/footer.component.html b/webapp/src/app/components/footer/footer.component.html index 1c155bf26..2d745befd 100644 --- a/webapp/src/app/components/footer/footer.component.html +++ b/webapp/src/app/components/footer/footer.component.html @@ -1,6 +1,6 @@
- Copyright © Social Computing Group 2024 + Copyright © Social Computing Group 2025
About diff --git a/webapp/src/app/components/sidebar/sidebar.component.css b/webapp/src/app/components/sidebar/sidebar.component.css index ccb9801a3..12e1b0d54 100644 --- a/webapp/src/app/components/sidebar/sidebar.component.css +++ b/webapp/src/app/components/sidebar/sidebar.component.css @@ -1,8 +1,8 @@ -@tailwind base; +/* @tailwind base; @tailwind components; -@tailwind utilities; +@tailwind utilities; */ -@layer components { +/* @layer components { */ .sidebar-icon { @apply relative flex items-center justify-center h-14 w-14 my-1.5 mx-auto bg-sky-700 text-white hover:bg-sky-900 rounded-3xl shadow-lg hover:rounded-md transition-all cursor-pointer; } @@ -25,8 +25,8 @@ .icon-button-badge { @apply absolute top-10 right-[-6px] w-6 h-6 flex justify-center bg-orange-600 text-white rounded-full shadow-lg; } -} - +/* +} */ .sidebar-icon.show-activity-indicator::after { content: ''; position: absolute; diff --git a/webapp/src/app/course-welcome/course-welcome.component.ts b/webapp/src/app/course-welcome/course-welcome.component.ts index 738a90dba..1cb8f19d9 100644 --- a/webapp/src/app/course-welcome/course-welcome.component.ts +++ b/webapp/src/app/course-welcome/course-welcome.component.ts @@ -156,10 +156,16 @@ export class CourseWelcomeComponent implements OnInit, CanComponentDeactivate { } this.topicChannelService.fetchTopics(course._id).subscribe((res) => { this.selectedCourse = res.course; - this.Users = course.users; - + this.Users = res.course.users; + //console.log('Users in course:', this.Users); + let userModerator = this.Users.find( + (user) => user.role.name === 'moderator' + ); + if (!userModerator) { + throw new Error(`Moderator not found for course with id ${course._id}`); + } // TODO: Bad implementation to get the moderator, i.e., course.users[0].userId - this.buildCardInfo(course.users[0].userId, course); + this.buildCardInfo(userModerator.userId, course); }); this.sanitizeDescription(this.courseDescription); diff --git a/webapp/src/app/models/croForm.ts b/webapp/src/app/models/croForm.ts new file mode 100644 index 000000000..c49e5127c --- /dev/null +++ b/webapp/src/app/models/croForm.ts @@ -0,0 +1,192 @@ +import { ArticleElementModel } from "../pages/components/knowledge-graph/articles/models/article-element.model" +import { VideoElementModel } from "../pages/components/knowledge-graph/videos/models/video-element.model" + +export interface ActivatorPartCRO { + resetFormStatus: boolean, + modelStatus: boolean, + vennDiagramStatus: boolean +} + +export interface Concept { + final_embedding?: string, + initial_embedding?: string, + name?: string, + rank?: number, + weight?: number, + mid?: string, + abstract?: string, + wikipedia?: string, + type?: string, + uri?: string, + cid?: string, + id?: string, + status?: string, + visible?: string +} + +export interface Neo4jResult { + records: Concept[] +} + +export interface FactorWeight { + similarity_score?: number, + creation_date?: number, + views?: number, + like_count?: number + user_rating?: number, + saves_count?: number, +} + +export interface ResourceNode { + abstract?: string, + author_image_url?: string, + author_name?: string, + bookmarked_count?: number, + composite_score?: number, // removed + description?: string, + description_full?: string, + duration?: string, + helpful_count?: number, + id?: number, + keyphrases?: string[], + labels?: string[], + like_count?: number, + not_helpful_count?: number + post_date?: string, + publish_time?: string, + rid?: string, + similarity_score?: number, + text?: string, + thumbnail?: string, + title?: string, + uri?: string, + views?: string, +} + +export interface ResourcesPagination { + concepts?: Concept[], + nodes: { + articles: paginatorArticleDetail, + videos: paginatorVideoDetail + }, + recommendation_type: string +} + +export interface RatingResource { + voted?: string, + helpful_count?: number, + not_helpful_count?: number +} + +export interface UserResourceFilterParamsResult { + cids?: { cid: string, name: string} [], + mids?: { mid: string, name: string} [], + slider_numbers?: { name: string} [] +} + +export interface UserResourceFilterResult { + articles: ArticleElementModel[], + videos: VideoElementModel[] +} + +export interface paginatorVideoDetail { + current_page: number, + total_pages: number, + total_items: number, + content: VideoElementModel[], +} + +export interface paginatorArticleDetail { + current_page: number, + total_pages: number, + total_items: number, + content: ArticleElementModel[] +} + + + + + +/* + +// export interface ResourceNodesContentVideo { +// node_id: number, +// } + +// export interface ResourceNodesContentArticle { +// node_id: number, +// } + +// export interface ResourceNodes { +// current_page: number, +// total_pages: number, +// total_items: number, +// content: VideoElementModel[] | ArticleElementModel[], +// } + +// export interface ResourcesPagination { +// algorithm_model: number, +// cro_form?: any, +// concepts?: any, +// edges?: number, +// nodes: ResourceNodes, +// } + +export interface RecommendationNodesVideo { + current_page: number, + total_pages: number, + total_items: number, + content: VideoElementModel[], +} +export interface RecommendationNodesArticle { + current_page: number, + total_pages: number, + total_items: number, + content: ArticleElementModel[], +} +export interface RecommendationNodes { + videos: RecommendationNodesVideo, + articles: RecommendationNodesArticle +} +export interface RecommendationTypePagination { + nodes: RecommendationNodes, // VideoElementModel[] | ArticleElementModel[], +} +export interface ResourcesPagination { + cro_form?: any, + concepts?: Concept[], + edges?: any, + recommendation_type_1: RecommendationTypePagination, + recommendation_type_2: RecommendationTypePagination, + recommendation_type_3: RecommendationTypePagination, + recommendation_type_4: RecommendationTypePagination, +} +*/ + + +export interface ModelRec { + model_1: boolean, + model_2: boolean, + model_3: boolean, + model_4: boolean +} + +export interface RecommendationAlgorithm { + status: boolean, + models: ModelRec +} + +export interface CROform { + category?: string, + concepts?: Concept[], + recommendation_algorithm?: RecommendationAlgorithm, + vennDiagramStatus?: boolean, + countOriginal?: number +} + +export class CROForm { + category: string; + concepts: Concept[]; + recommendation_algorithm: RecommendationAlgorithm; + vennDiagramStatus: boolean; + countOriginal?: number +} \ No newline at end of file diff --git a/webapp/src/app/pages/components/annotations/pdf-annotation/pdf-main-annotation/pdf-main-annotation.component.html b/webapp/src/app/pages/components/annotations/pdf-annotation/pdf-main-annotation/pdf-main-annotation.component.html index a3bff7c65..650ef6bf6 100644 --- a/webapp/src/app/pages/components/annotations/pdf-annotation/pdf-main-annotation/pdf-main-annotation.component.html +++ b/webapp/src/app/pages/components/annotations/pdf-annotation/pdf-main-annotation/pdf-main-annotation.component.html @@ -23,9 +23,11 @@ [show-all]="false" [page]="currentPDFPage$ | async" [zoom]="pdfZoom$ | async" + (error)="onPdfLoadError($event)" > + ; totalPages: number; docURL!: string; + subs = new Subscription(); private API_URL = environment.API_URL; private isInitialLoad: boolean = true; @@ -154,7 +156,7 @@ export class PdfMainAnnotationComponent implements OnInit, OnDestroy { private socket: Socket, private changeDetectorRef: ChangeDetectorRef, private slideKgGenerator: SlideKgOrderedService, - protected router: Router + protected router: Router, ) { this.getDocUrl(); this.store.dispatch(AnnotationActions.loadAnnotations()); @@ -365,10 +367,21 @@ export class PdfMainAnnotationComponent implements OnInit, OnDestroy { getDocUrl() { this.pdfViewService.currentDocURL.subscribe((url) => { this.docURL = this.API_URL + url.replace(/\\/g, '/'); - console.log('this.docURL', this.docURL); }); } - + onPdfLoadError(error: any) { + //console.error('PDF Load Error:', error); + const materialId = this.extractMaterialIdFromUrl(this.docURL); + if (materialId) { + this.pdfViewService.emitError(materialId); + } + + } + extractMaterialIdFromUrl(url: string): string | null { + // Example: /api/public/uploads/pdfs/6886010b1152d6b2ee79e612.pdf + const match = url.match(/\/([\w\d]+)\.pdf$/); + return match ? match[1] : null; + } pagechanging(e: any) { this.store.dispatch( AnnotationActions.setCurrentPdfPage({ pdfCurrentPage: e.first + 1 }) diff --git a/webapp/src/app/pages/components/knowledge-graph/articles/card-article-list/card-article-list.component.html b/webapp/src/app/pages/components/knowledge-graph/articles/card-article-list/card-article-list.component.html index a1d1bcc35..1450fe601 100644 --- a/webapp/src/app/pages/components/knowledge-graph/articles/card-article-list/card-article-list.component.html +++ b/webapp/src/app/pages/components/knowledge-graph/articles/card-article-list/card-article-list.component.html @@ -1,9 +1,13 @@ -
- -
+
+ + +
\ No newline at end of file diff --git a/webapp/src/app/pages/components/knowledge-graph/articles/card-article-list/card-article-list.component.ts b/webapp/src/app/pages/components/knowledge-graph/articles/card-article-list/card-article-list.component.ts index 6113144bc..ab2166342 100644 --- a/webapp/src/app/pages/components/knowledge-graph/articles/card-article-list/card-article-list.component.ts +++ b/webapp/src/app/pages/components/knowledge-graph/articles/card-article-list/card-article-list.component.ts @@ -13,5 +13,11 @@ export class CardArticleListComponent { @Input() public notUnderstoodConcepts: string[]; public article!: ArticleElementModel; + @Input() userId: string; + @Input() resultTabType: string = ""; @Input() currentMaterial?: Material; + + onResourceRemovedEvent(rid: string) { + this.articleElements = this.articleElements.filter(article => article.rid !== rid); + } } diff --git a/webapp/src/app/pages/components/knowledge-graph/articles/card-article/card-article.component.html b/webapp/src/app/pages/components/knowledge-graph/articles/card-article/card-article.component.html index 49cd3c0c4..12f7f29d0 100644 --- a/webapp/src/app/pages/components/knowledge-graph/articles/card-article/card-article.component.html +++ b/webapp/src/app/pages/components/knowledge-graph/articles/card-article/card-article.component.html @@ -1,130 +1,130 @@ -
-
- -

- {{ article.title }} -

-

- {{ article.title.substring(0, TITLE_MAX_LENGTH) + "..." }} -

- - Similarity Score: {{ article.similarity_score | percent }} - - -
-

Keyphrases:

-
- {{ concept }} -
-
-
-

- Abstract: -

-

- {{ article.abstract }} -

-

- {{ article.abstract.substring(0, ABSTRACT_MAX_LENGTH) }}... - - - Expand + +

+
+ +

-

-

- {{ article.abstract }} - +

- - Collapse -

-

-
-
-
- -
+ {{ article.title.substring(0, TITLE_MAX_LENGTH) + "..." }} + +
+
+
+
Similarity score:
+
{{ article?.similarity_score | percent }}
+
+ +
+
+ +
+
+ +
+
+ +
+
+
-
-
-
-
+ + +
+

Keyphrases:

+
{{concept}}
+
+
+

Abstract:

+

{{article.abstract}}

+

+ {{article.abstract.substring(0, ABSTRACT_MAX_LENGTH)}}... - + xmlns="http://www.w3.org/2000/svg" + width="16" + height="16" + fill="currentColor" + class="bi bi-plus-square" + viewBox="0 0 16 16"> + + - View on Wikipedia - + Expand +

+

+ {{article.abstract}} + Collapse +

+
+
+
+
+ + +
+ +
+
+
+ +
-
-
+
\ No newline at end of file diff --git a/webapp/src/app/pages/components/knowledge-graph/articles/card-article/card-article.component.ts b/webapp/src/app/pages/components/knowledge-graph/articles/card-article/card-article.component.ts index a1800e965..535be169c 100644 --- a/webapp/src/app/pages/components/knowledge-graph/articles/card-article/card-article.component.ts +++ b/webapp/src/app/pages/components/knowledge-graph/articles/card-article/card-article.component.ts @@ -12,6 +12,7 @@ import { getCurrentPdfPage } from '../../../annotations/pdf-annotation/state/ann import { ArticleElementModel } from '../models/article-element.model'; import { OverlayPanel } from 'primeng/overlaypanel'; import { DomSanitizer } from '@angular/platform-browser'; +import { MessageService } from 'primeng/api'; import { MaterialsRecommenderService } from 'src/app/services/materials-recommender.service'; import { Material } from 'src/app/models/Material'; import { Store } from '@ngrx/store'; @@ -27,6 +28,7 @@ export class CardArticleComponent { constructor( private sanitizer: DomSanitizer, private materialsRecommenderService: MaterialsRecommenderService, + private messageService: MessageService, private store: Store ) { // Subscribe to get the current PDF page from store @@ -42,6 +44,8 @@ export class CardArticleComponent { @Input() public notUnderstoodConcepts: string[]; @Output() onClick: EventEmitter = new EventEmitter(); + @Input() userId: string; + @Input() currentMaterial?: Material; subscriptions: Subscription = new Subscription(); // Manage subscriptions ABSTRACT_MAX_LENGTH = 600; @@ -51,6 +55,14 @@ export class CardArticleComponent { selectedConcepts: string[] = []; userCanExpand = true; + isDescriptionFullDisplayed = false; + isBookmarkFill = false; + articleDescription = ""; + saveOrRemoveParams = {"user_id": "", "rid": "", "status": this.isBookmarkFill}; + saveOrRemoveStatus = false; + @Input() resultTabType: string = ""; + @Output() resourceRemovedEvent = new EventEmitter(); // take rid + ngOnInit(): void {} public openArticle(article: any): void { @@ -92,4 +104,62 @@ export class CardArticleComponent { } this.userCanExpand = !this.userCanExpand; } + + ngOnChanges() { + this.isBookmarkFill = this.article?.is_bookmarked_fill; + this.saveOrRemoveParams.user_id = this.userId; + this.saveOrRemoveParams.rid = this.article?.rid; + } + + showDescriptionFull() { + this.isDescriptionFullDisplayed = this.isDescriptionFullDisplayed === true ? false : true; + } + + addToBookmark() { + this.isBookmarkFill = this.isBookmarkFill === true ? false : true; + this.saveOrRemoveParams.status = this.isBookmarkFill; + this.SaveOrRemoveUserResource(this.saveOrRemoveParams); + this.onResourceRemovedEvent(); + } + + saveOrRemoveBookmark() { + // detail: 'Open your Bookmark List to find this article' + if (this.isBookmarkFill == true) { + if (this.saveOrRemoveStatus === true) { + this.messageService.add({ key: 'resource_bookmark_article', severity: 'success', summary: '', detail: 'Article saved successfully'}); + } + } else { + if (this.saveOrRemoveStatus === false) { + this.messageService.add({key: 'resource_bookmark_article', severity: 'info', summary: '', detail: 'Article removed from saved'}); + } + } + } + + SaveOrRemoveUserResource(params) { + this.materialsRecommenderService.SaveOrRemoveUserResource(params) + .subscribe({ + next: (data: any) => { + if (data["msg"] == "saved") { + this.saveOrRemoveStatus = true; + this.article.is_bookmarked_fill = true; + } else { + this.saveOrRemoveStatus = false; + this.article.is_bookmarked_fill = false; + } + this.saveOrRemoveBookmark(); + }, + error: (err) => { + console.log(err); + this.saveOrRemoveStatus = false; + this.article.is_bookmarked_fill = false; + }, + } + ); + } + + onResourceRemovedEvent() { + if (this.isBookmarkFill === false && this.resultTabType === "saved") { + this.resourceRemovedEvent.emit(this.article.rid); + } + } } diff --git a/webapp/src/app/pages/components/knowledge-graph/articles/mocks/article.mock.ts b/webapp/src/app/pages/components/knowledge-graph/articles/mocks/article.mock.ts index bcbfa23d9..f6628ed9e 100644 --- a/webapp/src/app/pages/components/knowledge-graph/articles/mocks/article.mock.ts +++ b/webapp/src/app/pages/components/knowledge-graph/articles/mocks/article.mock.ts @@ -13,8 +13,8 @@ export const ArticleMock: ArticleElementModel[] = [ post_date: '10th August 2020', uri: 'https://de.wikipedia.org/wiki/Wikipedia:Hauptseite', similarity_score: 0.95, - helpful_counter: 9, - not_helpful_counter: 2 + helpful_count: 9, + not_helpful_count: 2 }, { id: 2, @@ -28,8 +28,8 @@ export const ArticleMock: ArticleElementModel[] = [ post_date: '10th August 2020', uri: 'https://de.wikipedia.org/wiki/Wikipedia:Kontakt', similarity_score: 0.90, - helpful_counter: 5, - not_helpful_counter: 0 + helpful_count: 5, + not_helpful_count: 0 }, { id: 3, @@ -43,8 +43,8 @@ export const ArticleMock: ArticleElementModel[] = [ post_date: '10th August 2020', uri: 'https://de.wikibooks.org/wiki/Wikibooks:Chat', similarity_score: 0.90, - helpful_counter: 4, - not_helpful_counter: 2 + helpful_count: 4, + not_helpful_count: 2 }, { id: 4, @@ -58,8 +58,8 @@ export const ArticleMock: ArticleElementModel[] = [ post_date: '10th August 2020', uri: 'https://de.wikibooks.org/wiki/Wikibooks:Administratoren', similarity_score: 0.87, - helpful_counter: 2, - not_helpful_counter: 0 + helpful_count: 2, + not_helpful_count: 0 }, { id: 5, @@ -73,8 +73,8 @@ export const ArticleMock: ArticleElementModel[] = [ post_date: '10th August 2020', uri: 'https://de.wikibooks.org/wiki/Wikibooks:Portal', similarity_score: 0.85, - helpful_counter: 1, - not_helpful_counter: 0 + helpful_count: 1, + not_helpful_count: 0 }, { id: 6, @@ -88,8 +88,8 @@ export const ArticleMock: ArticleElementModel[] = [ post_date: '10th August 2020', uri: 'https://wikisource.org/wiki/Wikisource:Community_Portal', similarity_score: 0.7, - helpful_counter: 3, - not_helpful_counter: 2 + helpful_count: 3, + not_helpful_count: 2 }, { id: 7, @@ -103,8 +103,8 @@ export const ArticleMock: ArticleElementModel[] = [ post_date: '10th August 2020', uri: 'https://de.wikipedia.org/wiki/Wikipedia:Autorenportal', similarity_score: 0.69, - helpful_counter: 4, - not_helpful_counter: 4 + helpful_count: 4, + not_helpful_count: 4 }, { id: 8, @@ -118,7 +118,7 @@ export const ArticleMock: ArticleElementModel[] = [ post_date: '10th August 2020', uri: 'https://de.wikipedia.org/w/index.php?title=Wikipedia:Hauptseite&action=info', similarity_score: 0.5, - helpful_counter: 0, - not_helpful_counter: 0 + helpful_count: 0, + not_helpful_count: 0 }, ]; diff --git a/webapp/src/app/pages/components/knowledge-graph/articles/models/article-element.model.ts b/webapp/src/app/pages/components/knowledge-graph/articles/models/article-element.model.ts index c7cf5c658..ab9dbc69b 100644 --- a/webapp/src/app/pages/components/knowledge-graph/articles/models/article-element.model.ts +++ b/webapp/src/app/pages/components/knowledge-graph/articles/models/article-element.model.ts @@ -9,6 +9,11 @@ export interface ArticleElementModel { author_name?: string; uri: string; similarity_score: number; - helpful_counter: number; - not_helpful_counter: number; + helpful_count: number; + not_helpful_count: number; + bookmarked_count?: number, + like_count?: number, + rid?: string, + updated_at?: string, + is_bookmarked_fill?: boolean } diff --git a/webapp/src/app/pages/components/knowledge-graph/concept-map/concept-map.component.css b/webapp/src/app/pages/components/knowledge-graph/concept-map/concept-map.component.css index ab16e287f..1580c6225 100644 --- a/webapp/src/app/pages/components/knowledge-graph/concept-map/concept-map.component.css +++ b/webapp/src/app/pages/components/knowledge-graph/concept-map/concept-map.component.css @@ -290,10 +290,10 @@ max-height: 30%; overflow-y: auto; } -:host ::ng-deep #recommendationButton { +/* :host ::ng-deep #recommendationButton { position: absolute; - /* white-space: normal; */ - /* word-wrap: break-word; */ + /* white-space: normal; + /* word-wrap: break-word; color: #ffffff; background: #eb590d; border: #eb590d; @@ -328,7 +328,7 @@ left: 1; margin-top: 0.5rem; z-index: 12; -} +} */ #conceptsCategoryButton { position: absolute; margin-top: 0.5rem; @@ -499,7 +499,6 @@ .p-dialog-content { display: flex; flex-direction: column; - } #materialKgControlPanel { @@ -510,6 +509,77 @@ padding-right: 1rem; float: right; } + +:host ::ng-deep #view_dnus .p-accordion-tab { + border: 2px solid #0288d1; +} + +:host ::ng-deep .p-accordion-header.accordionTab1 .p-accordion-header.accordionTab2 { + font-size: 12px; + background-color: #e9ecef; + color: #747d84; +} + +:host ::ng-deep .p-sidebar-header{ + display: none; +} + +:host ::ng-deep .p-sidebar-footer{ + display: none; +} + +:host ::ng-deep #recommendationButton { + background: #eb590d; + border: #eb590d; + border-radius: 30px; +} + +:host ::ng-deep .graphSection { + /* width: 72em; */ +} + +#hideNonUnderstoodPanel { + /* position: absolute; + left: 612px; + float: right; + top: 50%; + z-index: 12; */ + color: #ffffff; + background: #A8A9AA; + border-radius: 16px; + padding-top: 0.5rem; + padding-bottom: 0.5rem; + padding-left: 0.125rem; + padding-right: 0.125rem; + font-size: 2px; +} + +#showNonUnderstoodPanel { + position: absolute; + background-color: #ffffff; + color: #2196F3; + border: 1px solid #2196F3; + border-top-right-radius: 8px; + border-bottom-right-radius: 8px; + border-top-left-radius: 0; + border-bottom-left-radius: 0; + /* left: 1; + margin-top: 0.5rem; */ + margin-left: 1%; + z-index: 12; +} + +@media only screen and (max-width: 2200px) { + :host ::ng-deep .graphSection { + width: 72em; + } +} + +@media only screen and (max-width: 1800px) { + :host ::ng-deep .graphSection { + width: 110em; + } +} /* ::ng-deep .custom-close-btn { position: absolute; @@ -566,4 +636,4 @@ height: 500px; /* Default height */ height: 40px; padding: 8px 12px; -}*/ \ No newline at end of file +}*/ diff --git a/webapp/src/app/pages/components/knowledge-graph/concept-map/concept-map.component.html b/webapp/src/app/pages/components/knowledge-graph/concept-map/concept-map.component.html index ce55a6f41..5aaa0bd5d 100644 --- a/webapp/src/app/pages/components/knowledge-graph/concept-map/concept-map.component.html +++ b/webapp/src/app/pages/components/knowledge-graph/concept-map/concept-map.component.html @@ -558,6 +558,7 @@
+
@@ -635,349 +637,368 @@ iconPos="right" pTooltip="Show list of not understood concepts" class="p-button-secondary p-button-rounded" - > + > +
- - -

- Not Understood Concepts Lists -

- -
- - -

- Select not understood concept(s) from the graph -

-
-
+ +
+

+ Not Understood Concepts Lists +

+
+ + +
+ +
+ - - {{ concept.name }} - - -
-
- - -

- No not understood concepts so far! -

-
-
+ + + + + + + + + + + + +
+
+ +
+
+ - + No not understood concepts so far! + +
- {{ concept.name }} - - -
- - - -
- + > + + + + + + + + + + + + +
+ + +
+ +
+ + +
+ + +
- - - - +
- - +
- -
+ +
+ +
-
-
- -
-
-
-
- The knowledge graph for this learning material has not been - created yet. Once it’s created, you can view the Main Concepts - related to this slide here. -
-
-
- -
- +
- -
-
+
+ + +
+
-
-
- +
- -
-
-
-
-
- +
+ + +
+
+ + +
+
+ The knowledge graph for this learning material has not been + created yet. Once it’s created, you can view the Main Concepts + related to this slide here. +
+ + +
+
+ +
+
+ + +
+ + +
+
+ + +
+
+ + +
+ + +
+
+
- -
@@ -985,4 +1006,4 @@

- + \ No newline at end of file diff --git a/webapp/src/app/pages/components/knowledge-graph/concept-map/concept-map.component.ts b/webapp/src/app/pages/components/knowledge-graph/concept-map/concept-map.component.ts index ada25b644..c9f7ce11a 100644 --- a/webapp/src/app/pages/components/knowledge-graph/concept-map/concept-map.component.ts +++ b/webapp/src/app/pages/components/knowledge-graph/concept-map/concept-map.component.ts @@ -4,8 +4,9 @@ import { Output, EventEmitter, ChangeDetectorRef, - Renderer2, - + ViewChild, + Renderer2, + } from '@angular/core'; import { Store } from '@ngrx/store'; import { ConfirmationService, MenuItem, MessageService } from 'primeng/api'; @@ -30,6 +31,8 @@ import { getCurrentMaterial } from '../../materials/state/materials.reducer'; import { getCurrentPdfPage } from '../../annotations/pdf-annotation/state/annotation.reducer'; import { Socket } from 'ngx-socket-io'; import { FormControl, FormGroup, Validators } from '@angular/forms'; +import { ActivatorPartCRO, ResourcesPagination } from 'src/app/models/croForm'; +import { CustomRecommendationOptionComponent } from '../custom-recommendation-option/custom-recommendation-option.component'; import { getCurrentCourseId } from 'src/app/pages/courses/state/course.reducer'; import { CytoscapeComponent } from '../cytoscape/cytoscape.component'; @@ -154,10 +157,11 @@ export class ConceptMapComponent { courseIsEmpty?: boolean = undefined; recommendedConceptType = 'recommended_concept'; allSelected = false; + skippedFirst = true; tabs = [ { - label: 'Main Concepts', + label: 'Main Concepts', // graphSection command: (e) => { let tempMapData = this.filteredMapData; this.filteredMapData = null; @@ -189,36 +193,39 @@ export class ConceptMapComponent { } }, }, - { - label: 'Recommended Concepts', - icon: 'pi pi-fw pi-external-link', - disabled: true, - command: (e) => { - this.mainConceptsTab = false; - this.recommendedConceptsTab = true; - //Log the Activity User viewedrecommendedConcepts - this.logUserViewedRecommendedConcepts(); - //if navigating from materials tab - if (this.recommendedMaterialsTab) { - this.recommendedMaterialsTab = false; - //show sidebar on main tab - setTimeout(() => { - this.showConceptsList(); - }, 50); - } else { - this.recommendedMaterialsTab = false; - if (this.showConceptsListSidebar) { - setTimeout(() => { - this.showConceptsList(); - }, 1); - } else { - setTimeout(() => { - this.hideConceptsList(); - }, 1); - } - } - }, - }, + // { + // //label: '', + // //icon: 'pi pi-fw pi-external-link', + // //disabled: true, + // hidden: true, + // command: (e) => { + // // this.mainConceptsTab = false; + // // this.recommendedConceptsTab = false; + // // this.skippedFirst = true; + + // //Log the Activity User viewedrecommendedConcepts + // //this.logUserViewedRecommendedConcepts(); + // //if navigating from materials tab + // // if (this.recommendedMaterialsTab) { + // // this.recommendedMaterialsTab = false; + // // //show sidebar on main tab + // // setTimeout(() => { + // // this.showConceptsList(); + // // }, 50); + // // } else { + // // this.recommendedMaterialsTab = false; + // // if (this.showConceptsListSidebar) { + // // setTimeout(() => { + // // this.showConceptsList(); + // // }, 1); + // // } else { + // // setTimeout(() => { + // // this.hideConceptsList(); + // // }, 1); + // // } + // // } + // }, + // }, { label: 'Recommended Materials', icon: 'pi pi-fw pi-book', //changed the youtube icon to address violation @@ -259,6 +266,14 @@ export class ConceptMapComponent { currentPDFPage: number; private subscriptions: Subscription[] = []; + + activatorPartCRO: ActivatorPartCRO = { resetFormStatus: false, modelStatus: false, vennDiagramStatus: false}; + @ViewChild('croComponent', { static: false }) croComponent: CustomRecommendationOptionComponent; + resourcesPagination: ResourcesPagination = undefined; + isRecommendationButtonDisplayed = true; + conceptsUpdatedCRO: any; + + totalPages: any; constructor( private messageService: MessageService, //show toast messages @@ -309,11 +324,12 @@ export class ConceptMapComponent { this.kgNodes = null; this.recommendedConcepts = null; - this.tabs[1].disabled = true; - this.tabs[2].disabled = true; - this.kgTabsActivated = false; + this.tabs[1].disabled = true; + // this.tabs[2].disabled = true; + this.kgTabsActivated = true; this.filteredMapData = null; this.resultMaterials = null; + this.concepts1 = null; this.concepts2 = null; @@ -399,7 +415,7 @@ export class ConceptMapComponent { this.subscriptions.push( this.kgTabs.activateKgTabs().subscribe(() => { this.tabs[1].disabled = false; - this.tabs[2].disabled = false; + // this.tabs[2].disabled = false; this.kgTabsActivated = true; }) ); //Activate tabs @@ -456,6 +472,7 @@ export class ConceptMapComponent { ); this.subscriptions.push( slideConceptservice.didNotUnderstandConcepts.subscribe((res) => { + this.croUpdater(res, undefined); this.didNotUnderstandConceptsObj = res; this.didNotUnderstandConceptsNames = this.didNotUnderstandConceptsNames.map((concept) => concept.name); @@ -617,6 +634,9 @@ export class ConceptMapComponent { this.selectedTopConcepts = this.defaultTopConcepts; this.defaultTopConcepts = 15; this.resetFilter(); + + this.croUpdater(this.didNotUnderstandConceptsObj, this.previousConceptsObj); + } ngOnInit() { this.socket.on('log', this.printLogMessage); @@ -643,6 +663,7 @@ export class ConceptMapComponent { this.conceptFromChipObj ); this.conceptFromChipObj = null; + this.croUpdater(this.didNotUnderstandConceptsObj, this.previousConceptsObj); }, }, { @@ -658,6 +679,7 @@ export class ConceptMapComponent { ); this.slideConceptservice.updateNewConcepts(this.conceptFromChipObj); this.conceptFromChipObj = null; + this.croUpdater(this.didNotUnderstandConceptsObj, this.previousConceptsObj); }, }, ]; @@ -674,6 +696,7 @@ export class ConceptMapComponent { this.previousConceptFromChipObj ); this.previousConceptFromChipObj = null; + this.croUpdater(this.didNotUnderstandConceptsObj, this.previousConceptsObj); }, }, { @@ -688,6 +711,7 @@ export class ConceptMapComponent { this.previousConceptFromChipObj ); this.previousConceptFromChipObj = null; + this.croUpdater(this.didNotUnderstandConceptsObj, this.previousConceptsObj); }, }, ]; @@ -698,6 +722,8 @@ export class ConceptMapComponent { conceptName: new FormControl(null), conceptSlides: new FormControl(null), }); + + this.setResponsiveWidthKnowledgeGraph(); } ngAfterViewChecked() { @@ -778,9 +804,20 @@ export class ConceptMapComponent { if (this.materialKgActivated && !this.showMaterialKg) { this.materialKgActivated = false; - } + } } + onActiveItemChange(event: MenuItem) { + // console.warn("tab onActiveItemChange") // graphSection + if (event.label === 'Main Concepts') { + // console.warn("Main Concepts") + } else if (event.label === 'Recommended Concepts') { + // console.warn("Recommended Concepts") + } else { + } + + } + onResize(e) { if (this.showSlideKg) { try { @@ -799,8 +836,23 @@ export class ConceptMapComponent { this.changeDetectorRef.detectChanges(); } this.cyWidth = window.innerWidth * 0.9; + + this.setResponsiveWidthKnowledgeGraph() + } + + setResponsiveWidthKnowledgeGraph() { + console.warn("window.innerWidth ", window.innerWidth) + let knowledgeGraph = document.getElementById('graphSection'); + if (knowledgeGraph && knowledgeGraph.style) { + if (window.innerWidth < 2700) { + knowledgeGraph.style.width = '75%'; + } else if (window.innerWidth > 2700) { + knowledgeGraph.style.width = '85%'; + } + } } + //? This is responsible for setting the chip concept from the first section of the not understood concept list setChipConcept(concept: any): void { this.conceptFromChipObj = { @@ -837,11 +889,16 @@ export class ConceptMapComponent { if (flexboxNotUnderstood) { this.slideKgWidth = slideKgDialogDiv.offsetWidth - flexboxNotUnderstood.offsetWidth; - knowledgeGraph.style.marginLeft = 1 + 'rem'; + + if (knowledgeGraph) { + knowledgeGraph.style.marginLeft = 1 + 'rem'; + } } else { this.slideKgWidth = slideKgDialogDiv.offsetWidth; } - knowledgeGraph.style.width = this.slideKgWidth + 'px'; + // knowledgeGraph.style.width = this.slideKgWidth + 'px'; + this.setResponsiveWidthKnowledgeGraph(); + }, 2); } // hide sidebar @@ -1390,6 +1447,7 @@ export class ConceptMapComponent { this.conceptMapData = conceptsList; } async showRecommendations() { + this.resourcesPagination = null; if (this.disableShowRecommendationsButton) { this.infoToast(); } else { @@ -1399,9 +1457,9 @@ export class ConceptMapComponent { this.conceptMapRecommendedData = this.recommendedConcepts; this.filteredMapRecData = this.conceptMapRecommendedData; this.tabs[1].disabled = true; - this.tabs[2].disabled = true; + // this.tabs[2].disabled = true; this.kgTabsActivated = false; - + this.understoodConceptsObj.forEach((concept) => { this.allUnderstoodConcepts.push(concept.cid); }); @@ -1426,6 +1484,7 @@ export class ConceptMapComponent { } catch (err) { console.error(err); } + this.tabIndex = 2; this.showRecommendationButtonClicked = true; // this.callRecommendationsService.showRecommendationsClicked(); @@ -1436,6 +1495,7 @@ export class ConceptMapComponent { const reqDataMaterial1 = await this.getRecommendedMaterialsPerSlideMaterial(); + reqDataMaterial1["userId"] = this.userid; this.materialsRecommenderService .getRecommendedConcepts( @@ -1463,97 +1523,43 @@ export class ConceptMapComponent { }, 1); } - this.kgTabs.kgTabsEnable(); - this.mainConceptsTab = false; - this.recommendedConceptsTab = true; - //receive recommended concepts - //Log the activity User viewed all recommended concepts - this.logUserViewedRecommendedConcepts(); - // this.tabs[2].disabled = true; - this.recommendedMaterialsTab = false; - //////////////////////////call material-recommender///////////////////////// - this.materialsRecommenderService - .getRecommendedMaterials(reqData) //req data will be sent to the backend to search for materials based on not understood concepts - .subscribe({ - next: (result) => { - this.resultMaterials = result; - this.concepts1 = this.resultMaterials.concepts; - // ! Here is the problem this.resultMaterials.concepts includes just cid, id, name, weight. We also nee the type of each concept for the logging. - // Problem solved - this.concepts1.forEach((el, index, array) => { - let matchedConcept = this.didNotUnderstandConceptsObj.find( - (concept) => concept.id.toString() === el.id.toString() - ); - - if (matchedConcept) { - el.status = 'notUnderstood'; - el.type = matchedConcept.type; // Assigning type - } else { - matchedConcept = this.previousConceptsObj.find( - (concept) => - concept.cid.toString() === el.cid.toString() - ); - - if (matchedConcept) { - el.status = 'notUnderstood'; - el.type = matchedConcept.type; // Assigning type - } else { - matchedConcept = this.understoodConceptsObj.find( - (concept) => - concept.id.toString() === el.id.toString() - ); - - if (matchedConcept) { - el.status = 'understood'; - el.type = matchedConcept.type; // Assigning type - } else { - matchedConcept = this.newConceptsObj.find( - (concept) => - concept.id.toString() === el.id.toString() - ); - if (matchedConcept) { - el.status = 'unread'; - el.type = matchedConcept.type; // Assigning type - } - } - } - } - - array[index] = el; // Update the array element - }); - - // this.concepts1.forEach((el, index, array) => { - // if ( - // this.didNotUnderstandConceptsObj.some( - // (concept) => concept.id.toString() === el.id.toString() - // ) - // ) { - // el.status = 'notUnderstood'; - // array[index] = el; - // } else if ( - // this.previousConceptsObj.some( - // (concept) => - // concept.cid.toString() === el.cid.toString() - // ) - // ) { - // el.status = 'notUnderstood'; - // array[index] = el; - // } else if ( - // this.understoodConceptsObj.some( - // (concept) => concept.id.toString() === el.id.toString() - // ) - // ) { - // el.status = 'understood'; - // array[index] = el; - // } else { - // el.status = 'unread'; - // array[index] = el; - // } - // }); - - this.resultMaterials = this.resultMaterials.nodes; + //this.kgTabs.kgTabsEnable(); + //this.kgTabsActivated = true; + this.mainConceptsTab = false; + this.recommendedConceptsTab = false; + + //Log the activity User viewed all recommended concepts + this.logUserViewedRecommendedConcepts(); + //this.tabs[2].disabled = false; + //this.tabs[1].disabled = false; + // this.tabIndex = 2; + //this.tabs[1].disabled = true; + //this.recommendedMaterialsTab = true; + + //////////////////////////call material-recommender///////////////////////// + + + this.setHeightGraphComponent(); + this.isRecommendationButtonDisplayed = false; + let reqDataFinal = this.croComponent.buildFinalRequestRecMaterial(reqData); + + this.materialsRecommenderService + .getRecommendedMaterials(reqDataFinal) // reqData + .subscribe({ + next: (result) => { + this.isRecommendationButtonDisplayed = true; + this.resourcesPagination = result; this.kgTabs.kgTabsEnable(); + this.mainConceptsTab = false; + this.recommendedConceptsTab = false; + this.recommendedMaterialsTab = true; + // this.tabs[1].disabled = false; + //this.tabIndex = 2; + //this.tabs[2].disabled = false; + //this.kgTabsActivated = true; + + }, complete: () => { this.showRecommendationButtonClicked = false; @@ -1763,8 +1769,9 @@ export class ConceptMapComponent { this.kgNodes = null; this.recommendedConcepts = null; this.tabs[1].disabled = true; - this.tabs[2].disabled = true; + // this.tabs[2].disabled = true; this.kgTabsActivated = false; + this.filteredMapData = null; this.resultMaterials = null; @@ -1831,7 +1838,7 @@ export class ConceptMapComponent { this.recommendedConcepts = null; this.slideConceptservice.setUnderstoodConcepts(this.understoodConceptsObj); this.tabs[1].disabled = true; - this.tabs[2].disabled = true; + // this.tabs[2].disabled = true; this.kgTabsActivated = false; this.filteredMapData = null; this.selectedFilterValues = null; @@ -1936,7 +1943,7 @@ export class ConceptMapComponent { key: 'emptyList', severity: 'warn', summary: 'Empty Not Understood Concepts List', - detail: 'Select not understood concept(s) from the graph!', + detail: 'Mark some concept(s) as not understood first to get recommendations', }); } displayMessage(message: string): void { @@ -2294,6 +2301,37 @@ export class ConceptMapComponent { } } + croUpdater(didNotUnderstandConceptsObj: any[], previousConceptsObj: any[]) { + this.croComponent?.updateCROformAll(didNotUnderstandConceptsObj, previousConceptsObj); + } + + setHeightGraphComponent() { + let knowledgeGraph = document.getElementById('graphSection'); + if (knowledgeGraph) { + let ipo_interact = document.getElementById('ipo_interact'); + // console.warn("ipo_interact with -> ", ipo_interact.offsetWidth) + this.cyHeight = ipo_interact.offsetHeight - (ipo_interact.offsetHeight * 0.15); + } + } + + setWeightGraphComponent(event) { + setTimeout(() => { + let knowledgeGraph = document.getElementById('graphSection'); + if (this.showMaterialKg) { + console.warn("resize -> hideConceptsList -> HostListener event.screen ->", event.screen.width); + // let screenWidth = window.innerHeight; + let screenWidth = event.screen.width; // event.outerWidth + + if (screenWidth >= 768 && screenWidth < 992) { + knowledgeGraph.style.width = '40em'; + } + if (screenWidth > 992 && screenWidth <= 1200) { + knowledgeGraph.style.width = '40em'; + } + } + }, 3); + } + pagechanging(e: any) { this.currentPDFPage = e.page + 1; // Update the current page } @@ -2303,5 +2341,4 @@ export class ConceptMapComponent { event.stopPropagation(); window.open(url, '_blank'); } - } diff --git a/webapp/src/app/pages/components/knowledge-graph/custom-recommendation-option/custom-recommendation-option.component.css b/webapp/src/app/pages/components/knowledge-graph/custom-recommendation-option/custom-recommendation-option.component.css new file mode 100644 index 000000000..58e546f04 --- /dev/null +++ b/webapp/src/app/pages/components/knowledge-graph/custom-recommendation-option/custom-recommendation-option.component.css @@ -0,0 +1,135 @@ +:host ::ng-deep #add_more_concept .p-ripple { + font-size: 0.7rem; + color: #0288d1; +} + + +:host ::ng-deep #search_concept_category_1 .p-dropdown { + width: 20em; +} + +:host ::ng-deep #search_concept_category_1 .p-dropdown .p-dropdown-label { + font-size: 12px; +} +:host ::ng-deep #search_concept_category_1 .p-dropdown .p-dropdown-item { + font-size: 12px; +} + +/* :host ::ng-deep #search_concept_category_3 .p-dropdown { + width: 20em; +} +:host ::ng-deep #search_concept_category_3 .p-dropdown .p-dropdown-label { + font-size: 12px; +} +:host ::ng-deep #search_concept_category_3 .p-dropdown .p-dropdown-item { + font-size: 12px; +} */ + + +:host ::ng-deep #search_concept_category_3 .p-multiSelect { + width: 20em; +} + + + +:host ::ng-deep .raning_progress_bar .p-progressbar .p-progressbar-label { + display: block !important; + font-size: 0.6rem; +} + +/* change checkbox color */ +:host ::ng-deep .p-checkbox .p-checkbox-box { + border: 2px solid #0288d1; +} +:host ::ng-deep .p-checkbox-box:hover { + border-color: #0288d1; +} +:host ::ng-deep .p-checkbox-box.p-highlight { + background: #0288d1; + border-color: #0288d1; +} + +/* change slider color */ +:host ::ng-deep .p-slider .p-slider-range { + background-color: #0288d1; +} +:host ::ng-deep .p-slider .p-slider-handle { + border: 2px solid #0288d1; + background-color: #0288d1; +} + +/* change radio button color */ +.p-radiobutton { + width: 16px !important; + height: 16px !important; +} +.p-radiobutton .p-radiobutton-box { + border: 2px solid #0288d1; + width: 16px !important; + height: 16px !important; + border-radius: 50% !important; +} +.p-radiobutton .p-radiobutton-box .p-radiobutton-icon { + background-color: #0288d1; + width: 8px !important; + height: 8px !important; + left: 50% !important; + top: 50% !important; + transform: translate(-50%, -50%) !important; +} +.p-radiobutton-box:hover { + border-color: #0288d1; +} +.p-radiobutton-box.p-highlight { + background: #0288d1; + border-color: #0288d1; +} + +.factor_weight_disabled { + pointer-events: none; + opacity: 0.3; +} + +/* info tooltip icon */ +/* :host ::ng-deep #cro_content .pi pi-info-circle { + font-size: 0.6rem; +} */ + +.raning_progress_bar_value { + position: absolute; + /* top: 0; */ + left: 50%; + transform: translateX(-50%); + width: 100%; + text-align: center; + font-weight: bold; + line-height: 30px; /* Match the height of the progress bar */ + color: #000; /* Set the text color */ + font-size: 1rem; +} + + + +/* :host ::ng-deep #cro_second .p-panel-title { + color: #2196F3; + font-size: 12px; +} + +:host ::ng-deep #cro_second .p-panel { + border: 2px solid #0288d1; +} + +:host ::ng-deep #cro_second .p-panel-content { + border-style: none !important; +} */ + + + +/* :host ::ng-deep #cro_sorting .p-panel-title { + color: #2196F3; + font-size: 12px; +} + +:host ::ng-deep #cro_sorting .p-panel { + border: 2px solid #0288d1; +} */ \ No newline at end of file diff --git a/webapp/src/app/pages/components/knowledge-graph/custom-recommendation-option/custom-recommendation-option.component.html b/webapp/src/app/pages/components/knowledge-graph/custom-recommendation-option/custom-recommendation-option.component.html new file mode 100644 index 000000000..9326f8e49 --- /dev/null +++ b/webapp/src/app/pages/components/knowledge-graph/custom-recommendation-option/custom-recommendation-option.component.html @@ -0,0 +1,293 @@ + + + +
+
+ +
+ + +
+ + Customize your recommendations + + +
+ +
+
+ +
+
Select concepts to be used for recommendation
+ +
+ +
+
+ + +
+
+
+ + +
+ +
+
+
+
+
+ + +
+ +
+
+
+
+
+ + +
+ +
+
+
+
+ + +
+ + +
+ + +
+
+
+ Concept {{i +1}} : {{ cro?.name }} +
+
+
weight
+
+
+
0
+ +
{{cro?.weight | number:'1.2-2'}}
+
1
+
+
+
+
+
+ + + + +
+
+
+ + + + + +
+
+
Show Less
+
+
+ +
+
+
+ +
+
Show More
+
+
+ +
+
+
+
+ + +
+
How should be the recommendations generated:
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+
Factors impacting the ranking of recommendations
+
+ +
+
+ +
+
Factors
+
Weights
+
Impact on ranking
+
+ +
+
+
+ +
+
+ + +
+ +
+ {{ factor.title }} +
+
+ +
+
+
+
+
0
+
{{ factor.value | number:'1.2-2' }}
+
1
+
+
+
+
+
+
+
+
+ + + +
+
+
+
+ +
+
+
+ +
+ +
+
+
diff --git a/webapp/src/app/pages/components/knowledge-graph/custom-recommendation-option/custom-recommendation-option.component.spec.ts b/webapp/src/app/pages/components/knowledge-graph/custom-recommendation-option/custom-recommendation-option.component.spec.ts new file mode 100644 index 000000000..f1f4433a0 --- /dev/null +++ b/webapp/src/app/pages/components/knowledge-graph/custom-recommendation-option/custom-recommendation-option.component.spec.ts @@ -0,0 +1,21 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { CustomRecommendationOptionComponent } from './custom-recommendation-option.component'; + +describe('CustomRecommendationOptionComponent', () => { + let component: CustomRecommendationOptionComponent; + let fixture: ComponentFixture; + + beforeEach(() => { + TestBed.configureTestingModule({ + declarations: [CustomRecommendationOptionComponent] + }); + fixture = TestBed.createComponent(CustomRecommendationOptionComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/webapp/src/app/pages/components/knowledge-graph/custom-recommendation-option/custom-recommendation-option.component.ts b/webapp/src/app/pages/components/knowledge-graph/custom-recommendation-option/custom-recommendation-option.component.ts new file mode 100644 index 000000000..9510c748a --- /dev/null +++ b/webapp/src/app/pages/components/knowledge-graph/custom-recommendation-option/custom-recommendation-option.component.ts @@ -0,0 +1,707 @@ +import { Component, Input, OnChanges, OnInit, SimpleChanges } from '@angular/core'; +import { MessageService } from 'primeng/api'; +import { BehaviorSubject } from 'rxjs'; +import { ActivatorPartCRO, CROform, Neo4jResult } from 'src/app/models/croForm'; +import { CustomRecommendationOptionService } from 'src/app/services/custom-recommenation-option.service'; + + +export interface factorWeights { + original: any[], + normalized: { + like_count?: number; + creation_date?: number; + views?: number; + similarity_score?: number; + saves_count?: number; + user_rating?: number; + } +}; + + +@Component({ + selector: 'app-custom-recommendation-option', + templateUrl: './custom-recommendation-option.component.html', + styleUrls: ['./custom-recommendation-option.component.css'] +}) +export class CustomRecommendationOptionComponent implements OnChanges, OnInit { + @Input() userId: string; + @Input() materialId: string; + @Input() slideId: number = 1; + @Input() didNotUnderstandConceptsObj: any [] = []; + @Input() previousConceptsObj: any [] = []; + @Input() activatorPartCRO: ActivatorPartCRO; + @Input() conceptsUpdated: any; + + isCustomRecOptionDisplayed = true; + cro_concept_weight: number; + cro_concept_selected: any | undefined; + croForm: any = { + user_id: null, + mid: null, + slide_id: null, + category: "1", + concepts: [], + recommendation_type: "1", + factor_weights: null, + pagination_params: { "page_number": 1, "page_size": 10 } + }; + + factor_weight_checked = true; + factor_weights: factorWeights = { + original : null, + normalized: null + } + + croFormBackup: CROform = {}; + numberConceptsToBeChecked = 0; + CROconceptsManually = []; // from the whole material + + CROconceptsManuallySelection1 = []; // from the whole slide + CROconceptsManuallySelection2 = []; // from the whole material + + isAddMoreConceptDisplayed = false; + isMoreThan5ConceptDisplayed = false; + seeMore = true; + tmpCroFormConcepts = []; + + croFormObj = new BehaviorSubject(this.croForm); + + // result component + resultTabSelected: number = 1; + factorsTabArticle = ["similarity_score", "saves_count", "user_rating"]; + deactivateAlgo3and4 = false; + + @Input() data: any[] = []; + @Input() DNUCurrent: any[] = []; + @Input() DNUPrevious: any[] = []; + + constructor( + private croService: CustomRecommendationOptionService, + private messageService: MessageService + ) { + this.croFormObj.next(this.croForm); + + this.factor_weights.original = [ + { + "title": "Similarity Score", + "key": "similarity_score", + "value": 0.169000934, + "checked": true, + "deactivated": false + }, + { + "title": "Creation Date on YouTube", + "key": "creation_date", + "value": 0.141456583, + "checked": true, + "deactivated": false + }, + { + "title": "No. of Views on YouTube", + "key": "views", + "value": 0.186741363, + "checked": true, + "deactivated": false + }, + { + "title": "No. of Likes on YouTube", + "key": "like_count", + "value": 0.177404295, + "checked": true, + "deactivated": false + }, + + { + "title": "User Rating on CourseMapper", + "key": "user_rating", + "value": 0.18627451, + "checked": true, + "deactivated": false + }, + { + "title": "No. of Saves on CourseMapper", + "key": "saves_count", + "value": 0.139122316, + "checked": true, + "deactivated": false + } + ]; + this.factor_weights.normalized = { + 'like_count': 0.177404295, + 'creation_date': 0.141456583, + 'views': 0.186741363, + 'similarity_score': 0.169000934, + 'saves_count': 0.139122316, + 'user_rating': 0.18627451 + } + } + + ngOnInit(): void { + localStorage.removeItem('resultTabSelected'); + this.croForm["user_id"] = this.userId; + this.croForm["mid"] = this.materialId; + this.croForm["slide_id"] = this.slideId; + // this.croService.setResultaTabValue("1"); + + this.croService.isResultTabSelected$.subscribe(tabSelected => { + if (tabSelected === '0') { + this.factor_weights.original.forEach(factor => { + factor.checked = true; + factor.deactivated = false; + }) + } + else if (tabSelected === '1') { + for (let factor of this.factor_weights.original) { + if (!this.factorsTabArticle.includes(factor.key) === true) { + factor.checked = false; + factor.deactivated = true; + } + } + } + }); + } + + updateConcpetAfterRecommandation() { + // this.croForm.concepts = this.conceptsUpdated; + for (let conceptU of this.conceptsUpdated) { + for (let concept of this.croForm.concepts) { + if (conceptU.cid === concept.cid) { + concept.weight = conceptU.weight; + } else { + concept.status = false; + } + } + } + } + + ngOnChanges(changes: SimpleChanges) { + for (const propName in changes) { + if (propName === "materialId") { + this.getConceptsManually(); + } + if (propName === "slideId") { + } + this.updateCROformAll(this.didNotUnderstandConceptsObj, this.previousConceptsObj); + } + } + + showCRO() { + this.isCustomRecOptionDisplayed = this.isCustomRecOptionDisplayed === true ? false : true; + } + + sortDictBykey(data) { + const sorted = Object.keys(data) + .sort() + .reduce((acc, key) => { + acc[key] = data[key]; + return acc; + }, {}); + + return sorted; + } + + + updateFactorWeight() { + let data = this.getFactorWeight(); + this.factor_weights.normalized = this.normalizeFactorWeights(data, [], "l1", true, true) as { like_count?: number; creation_date: number; views: number; similarity_score: number; saves_count: number; user_rating: number; };; + } + + getFactorWeight() { + let result = {}; + for (let factor of this.factor_weights.original) { + if (factor.checked === true) { + result[factor.key] = factor.value + } + } + + this.croForm.factor_weights = result; + // this.croForm.factor_weights.weights = result; + return result + } + + // showRecTypeAndFactorWeight() { + // this.croForm.factor_weights.status = this.croForm?.concepts.length > 0 ? true : false; + // } + + activateOrNotFactorWeight(event, factor_index: number) { + let factor_div = document.getElementById("factor_weight_"+factor_index); + + if (event.checked === true) { + factor_div.classList.remove('factor_weight_disabled'); + } else { + factor_div.classList.add('factor_weight_disabled'); + } + this.updateFactorWeight(); + } + + + debugCRO(name: string, obj, hide = false) { + if (hide) { + console.warn(`CRO -> ${name} -> `, obj) + } + } + + resetCROform() { + const user_id = this.croForm.user_id; + const mid = this.croForm.mid; + const category = this.croForm.category; + const slide_id = this.croForm.slide_id; + + this.croForm = { + user_id: user_id, + mid: mid, + slide_id: slide_id, + category: category, + concepts: [], + recommendation_type: "1", + factor_weights: this.factor_weights, + pagination_params: null + } + } + + getConceptsByCids(cids: string) { + this.croService.getConceptsByCids(this.userId, cids).subscribe((res: Neo4jResult) => { + for (let record of res.records) { + for (let concept of this.croForm.concepts) { + if (concept.cid === record.cid) { + concept["weight"] = record.weight; + break; + } + } + } + // this.croForm.concepts = res.records; + }); + } + + getConceptsManually() { + if (this.materialId) { + if (this.CROconceptsManually.length == 0) { + this.croService.getConceptsModifiedByUserIdAndMid(this.materialId, this.userId).subscribe((res: Neo4jResult) => { + this.CROconceptsManually = res.records; + }); + } + } + } + + resetConceptsList() { + this.CROconceptsManually.forEach(x => {if (x.status === true) x.status = false }); + this.CROconceptsManuallySelection1.forEach(x => {if (x.status === true) x.status = false }); + this.CROconceptsManuallySelection2.forEach(x => {if (x.status === true) x.status = false }); + // this.get_concepts_manually_current_slide(); + } + + updateNumberConceptsToBeChecked() { + if (this.croForm.concepts.length > 0) { + // sort concepts based on the weight + // let sortedConcepts = this.croForm.concepts.sort((a, b) => b.weight - a.weight); + // this.croForm.concepts = sortedConcepts; + + let count = 0; + for (let i = 0; i < this.croForm.concepts.length; i++) { + let node = this.croForm.concepts[i]; + if (node["status"] === true) { + count += 1; + } + + // check 5 default concepts + if (i < 5) { + // node["status"] = true; + } else if (i >= 5) { + node["status"] = false; + } + } + this.numberConceptsToBeChecked = 5 - count; + } + this.croFormObj.next(this.croForm); + + /** + organize the list by weigth + this.dynamicConceptAdded(); + + Sort by status + this.croForm.concepts.sort((a, b) => b.status - a.status); + + this.croForm.countOriginal = this.croForm.concepts.length; + */ + } + + addDictsIfNotExists(newDictsArray) { + newDictsArray.forEach(newDict => { + const exists = this.croForm.concepts.some(dict => dict.cid === newDict.cid); + if (!exists) { + newDict["status"] = true; + newDict["visible"] = true; + this.croForm.concepts.push(newDict); + } + }); + } + + mapConcept(cids: string[], conceptsList: any[]) { + this.croService.getConceptsByCids(this.userId, cids.toString()).subscribe((res: Neo4jResult) => { + this.croForm.concepts = res.records; + for(let concept of this.croForm.concepts) { + concept["status"] = true; + concept["visible"] = true; + } + }); + + /* + // deep copy of conceptsManuallyOriginal + // let conceptsManuallyOriginalCopied = JSON.parse(JSON.stringify(this.conceptsManuallyOriginal)); + let concepts = []; + for(let id of cids) { + for(let node of conceptsList) { + if (id === node.cid) { + node["status"] = true; + node["visible"] = true; + concepts.push(node); + } + } + this.croForm.concepts = concepts; + } + // this.dynamicConceptAdded(); + // this.updateNumberConceptsToBeChecked(); + */ + } + + areArraysEqualById(array1, array2) { + // Check if arrays have the same length + if (array1.length !== array2.length) { + return false; + } + + // Create Sets of IDs from both arrays + const idSet1 = new Set(array1.map(item => item.id)); + const idSet2 = new Set(array2.map(item => item.id)); + + // Check if the Sets have the same size + if (idSet1.size !== idSet2.size) { + return false; + } + + // Check if every ID in idSet1 is also in idSet2 + for (let id of idSet1) { + if (!idSet2.has(id)) { + return false; + } + } + + // If we've made it this far, the arrays are equal + return true; + } + + updateCROform(conceptsObj: any[], category: string) { + if (this.materialId) { + this.croForm.concepts = []; + let cids = []; + + if ((category === "1" || category === "2") && conceptsObj) { + cids = conceptsObj.map((x) => x.cid); + } + + if (category === "1") { + this.mapConcept(cids, this.CROconceptsManuallySelection1); + } else if (category === "2") { + this.mapConcept(cids, this.CROconceptsManuallySelection2); + } + + else if (category === "3") { + this.getConceptsManually(); + } + } + } + + updateCROformAll(didNotUnderstandConceptsObj, previousConceptsObj) { + if (this.croForm.category === "1") { + this.updateCROform(didNotUnderstandConceptsObj, "1"); + + } else if (this.croForm.category === "2") { + let previousDNU; + if (previousConceptsObj) { + previousDNU = previousConceptsObj; + } else { + previousDNU = this.previousConceptsObj; + } + this.updateCROform([...didNotUnderstandConceptsObj, ...previousDNU], "2"); + + } else if (this.croForm.category === "3") { + this.updateCROform(undefined, "3"); + + } else { + this.croForm.category === "1" + this.updateCROform(didNotUnderstandConceptsObj, "1"); + } + } + + combine2ConceptsList(arrayA, arrayB) { + arrayA.forEach(itemA => { + const existsInB = arrayB.some(itemB => itemB.cid === itemA.cid); + if (!existsInB) { + arrayB.push(itemA); + } + }); + return arrayA; + } + + setCROcategory(event) { + this.croForm.category = event.value; + this.resetCROform() + this.resetConceptsList(); + this.isAddMoreConceptDisplayed = false; + this.isMoreThan5ConceptDisplayed = false; + + if (this.croForm.category === "1") { + this.updateCROform(this.didNotUnderstandConceptsObj, "1"); + this.deactivateAlgo3and4 = false; + + } else if (this.croForm.category === "2") { + this.updateCROform([...this.didNotUnderstandConceptsObj, ...this.previousConceptsObj], "2"); + this.deactivateAlgo3and4 = true; + + } else if (this.croForm.category === "3") { + this.updateCROform(undefined, "3"); + this.deactivateAlgo3and4 = true; + } + } + + setStatus(event, cro) { + this.deactivateSelection(); + // this.updateNumberConceptsToBeChecked(); + } + + selectTopConcept() { + let sortedConcepts = []; + sortedConcepts = this.croForm.concepts.sort((a, b) => b.weight - a.weight); + this.croForm.concepts = sortedConcepts; // sortedConcepts.slice(0, 5); + } + + displayAddMoreConcept() { + this.isAddMoreConceptDisplayed = this.isAddMoreConceptDisplayed === true ? false : true; + } + + displaySeeMore() { + this.seeMore = this.seeMore === true ? false : true; + for (let i = 5; i < this.croForm.concepts.length; i++) { + // console.warn(this.croForm.concepts[i]) + + if (this.seeMore === false) { + this.croForm.concepts[i]["visible"] = false; + } else { + this.croForm.concepts[i]["visible"] = true; + } + } + } + + deactivateSelection() { + let conceptsWithStatusTrue = this.croForm.concepts.filter(x => x.status === true) + if (conceptsWithStatusTrue.length > 5) { + this.isMoreThan5ConceptDisplayed = true; + } else { + this.isMoreThan5ConceptDisplayed = false; + } + } + + setSelectManuallyOnChangeMultiSelect(event) { + let concept_selected = event.itemValue; + if (concept_selected) { + const index = this.croForm.concepts.findIndex(concept => concept.cid === concept_selected.cid); + + if (index !== -1) { + this.croForm.concepts.splice(index, 1); + } else { + concept_selected["status"] = true; + concept_selected["visible"] = true; + this.croForm.concepts.push(concept_selected); + } + } else { + if (event.value.length === this.CROconceptsManually.length) { + this.croForm.concepts = event.value; + this.croForm.concepts.forEach((concept) => {concept["status"]; concept["status"]}); + } else if (event.value.length === 0) { + this.croForm.concepts = []; + } + } + + this.croForm.concepts.sort((a, b) => { + if (a.name < b.name) { + return -1; + } + if (a.name > b.name) { + return 1; + } + return 0; + }); + this.deactivateSelection(); + } + + setSelectManuallyOnChange(event) { + this.deactivateSelection(); + + if (event.value) { + let value = event.value; + const conceptFound = this.croForm.concepts.find((concept) => concept.cid === value.cid); + + if (conceptFound === undefined) { + value["status"] = true; + value["visible"] = true; + this.croForm.concepts.push(value); + + // this.getConceptsModifiedByUserIdAndCids(); + } else { + for(let concept of this.croForm.concepts) { + if (value.cid === concept.cid) { + concept.status = !value.status; + concept.visible = !value.status; + break + } + } + } + } + // this.updateNumberConceptsToBeChecked(); + } + + removeSelectManuallyOnChange(index: number, cid: string) { + if (index > -1) { + this.croForm.concepts.splice(index, 1); + + for (let node of this.CROconceptsManually) { + if (cid === node.cid) { + node.status = false; + node.visible = false; + break; + } + } + } + + // this.updateNumberConceptsToBeChecked(); + this.messageService.add({ key: 'cro_concept_removed', severity: 'info', summary: '', detail: 'Concept removed'}); + } + + buildFinalRequestRecMaterial(reqData) { + let concepts = []; + for(let concept of this.croForm.concepts) { + if (concept.status === true) { + concepts.push(concept); + } + } + + this.getFactorWeight(); + let croFormRequest = { + default: reqData, + rec_params: JSON.parse(JSON.stringify(this.croForm)) + } + croFormRequest.rec_params.concepts = concepts; + + // Store CRO Request Params + localStorage.removeItem('resourcesPaginationParams'); + localStorage.setItem('resourcesPaginationParams', JSON.stringify(croFormRequest)); + + return croFormRequest; + } + + deactivateTabResultArticle(factorKey): void { + const resultTabSelected = localStorage.getItem('resultTabSelected'); + if (resultTabSelected) { + if (resultTabSelected === '1' && !this.factorsTabArticle.includes(factorKey) === true ) { + for (let factor of this.factor_weights.original) { + if (factor.key === factorKey) { + factor.checked = false; + break; + } + } + // return true; + } + } + } + + checkResultTabSelected(factorKey) { + const resultTabSelected = localStorage.getItem('resultTabSelected'); + if (resultTabSelected) { + if (resultTabSelected === '1' && !this.factorsTabArticle.includes(factorKey) === true ) { + for (let factor of this.factor_weights.original) { + if (factor.key === factorKey) { + factor.checked = false; + break; + } + } + return true; + } + } + for (let factor of this.factor_weights.original) { + if (factor.key === factorKey) { + factor.checked = true; + break; + } + } + return false; + } + + deactivateAlgorithmOption() { + if (this.croForm.category === '2' || this.croForm.category === '3') { + return true; + } + return false; + } + + + normalizeFactorWeights( + factorWeights: any, // Record = {}, + values: number[] = [], + methodType: string = 'l1', + complete: boolean = true, + sumValue: boolean = true + ) // : Record | number[] | {} + { + // console.info('Normalization of factor weights'); + if (!factorWeights || Object.keys(factorWeights).length === 0) { + return {}; + } + + let normalizedValues: number[] | null = null; + let scaledData: number[] | null = null; + + if (factorWeights) { + values = Object.values(factorWeights); + } + + switch (methodType) { + case 'l1': + const l1Sum = values.reduce((acc, val) => acc + Math.abs(val), 0); + normalizedValues = values.map((val) => +(val / l1Sum).toFixed(3)); + break; + + case 'l2': + const l2Sum = Math.sqrt(values.reduce((acc, val) => acc + val * val, 0)); + normalizedValues = values.map((val) => +(val / l2Sum).toFixed(3)); + break; + + case 'max': + const maxValue = Math.max(...values); + normalizedValues = values.map((val) => +(val / maxValue).toFixed(3)); + break; + + case 'min-max': + const minValue = Math.min(...values); + const range = Math.max(...values) - minValue; + scaledData = values.map((val) => +((val - minValue) / range).toFixed(3)); + normalizedValues = scaledData; + break; + } + + if (sumValue && normalizedValues) { + // console.info('Factor weight sum ->', normalizedValues.reduce((acc, curr) => acc + curr, 0)); + } + + if (complete && normalizedValues) { + const keyNames = Object.keys(factorWeights); + const res = keyNames.reduce((acc, key, index) => { + acc[key] = normalizedValues![index]; + return acc; + }, {} as Record); + return res; + } + + return normalizedValues || {}; + } + +} diff --git a/webapp/src/app/pages/components/knowledge-graph/cytoscape-recommended/cytoscape-recommended.component.ts b/webapp/src/app/pages/components/knowledge-graph/cytoscape-recommended/cytoscape-recommended.component.ts index f64fe8b0b..df5d7b1ee 100644 --- a/webapp/src/app/pages/components/knowledge-graph/cytoscape-recommended/cytoscape-recommended.component.ts +++ b/webapp/src/app/pages/components/knowledge-graph/cytoscape-recommended/cytoscape-recommended.component.ts @@ -255,7 +255,7 @@ export class CytoscapeRecommendedComponent { }; ngOnChanges() { - console.log(this.elements); + // console.log(this.elements); this.init(); } diff --git a/webapp/src/app/pages/components/knowledge-graph/cytoscape-slide/cytoscape-slide.component.ts b/webapp/src/app/pages/components/knowledge-graph/cytoscape-slide/cytoscape-slide.component.ts index b132e99cc..5c25c866a 100644 --- a/webapp/src/app/pages/components/knowledge-graph/cytoscape-slide/cytoscape-slide.component.ts +++ b/webapp/src/app/pages/components/knowledge-graph/cytoscape-slide/cytoscape-slide.component.ts @@ -371,7 +371,7 @@ export class CytoscapeSlideComponent implements OnInit, OnChanges { setTimeout(() => { let cy_container = this.renderer.selectRootElement('#cySlide'); - if (this.elements !== undefined) { + if (this.elements && this.elements !== undefined) { if (Object.keys(this.elements.nodes).length > 5) { let topElements = this.elements.nodes.filter( (node) => node.data.rank < 6 diff --git a/webapp/src/app/pages/components/knowledge-graph/knowledge-graph.module.ts b/webapp/src/app/pages/components/knowledge-graph/knowledge-graph.module.ts index b1a96750f..d75856118 100644 --- a/webapp/src/app/pages/components/knowledge-graph/knowledge-graph.module.ts +++ b/webapp/src/app/pages/components/knowledge-graph/knowledge-graph.module.ts @@ -37,6 +37,14 @@ import { DateAgoPipe } from './videos/pipes/date-ago.pipe'; import { LinkifyPipe } from './videos/pipes/linkify.pipe'; import { SafeHtmlPipe } from './videos/pipes/safehtml.pipe'; import {TabViewModule} from 'primeng/tabview'; +import { TooltipModule } from 'primeng/tooltip'; +import { SliderModule } from 'primeng/slider'; +import { CustomRecommendationOptionComponent } from './custom-recommendation-option/custom-recommendation-option.component'; +import { PanelModule } from 'primeng/panel'; +import { ScrollPanelModule } from 'primeng/scrollpanel'; +import { ListboxModule } from 'primeng/listbox'; +import { ProgressBarModule } from 'primeng/progressbar'; +import { CardModule } from 'primeng/card'; import { InputTextModule } from 'primeng/inputtext'; import { AutoCompleteModule } from 'primeng/autocomplete'; import { MultiSelectModule } from 'primeng/multiselect'; @@ -65,6 +73,7 @@ import { PaginatorModule } from 'primeng/paginator'; DateAgoPipe, LinkifyPipe, SafeHtmlPipe, + CustomRecommendationOptionComponent, ], imports: [ InputTextModule, @@ -86,7 +95,24 @@ import { PaginatorModule } from 'primeng/paginator'; RadioButtonModule, OverlayPanelModule, TabViewModule, + + // boby024 + RadioButtonModule, + TooltipModule, + CheckboxModule, + SliderModule, + FormsModule, + DropdownModule, ReactiveFormsModule, + PaginatorModule, + PanelModule, + ScrollPanelModule, + ListboxModule, + ProgressBarModule, + InputTextModule, + MultiSelectModule, + CardModule, + // ReactiveFormsModule, AutoCompleteModule, MultiSelectModule, PdfViewerModule, diff --git a/webapp/src/app/pages/components/knowledge-graph/rating/rating.component.html b/webapp/src/app/pages/components/knowledge-graph/rating/rating.component.html index 3270aa8d2..2d779e245 100644 --- a/webapp/src/app/pages/components/knowledge-graph/rating/rating.component.html +++ b/webapp/src/app/pages/components/knowledge-graph/rating/rating.component.html @@ -5,7 +5,7 @@ (click)="toggleOverlay($event, element, op)" [class.active]="isLiked"> - {{element.helpful_counter}} + {{element.helpful_count}} Helpful @@ -13,7 +13,7 @@ Not helpful @@ -22,11 +22,11 @@
-

Which of the following concepts did this article help you to understand?

+

Which of the following concepts did this video/article help you to understand?

+ label="{{concept.name}}" [(ngModel)]="selectedConcepts" (onChange)="onChangeConcept($event, concept?.cid)"> diff --git a/webapp/src/app/pages/components/knowledge-graph/rating/rating.component.ts b/webapp/src/app/pages/components/knowledge-graph/rating/rating.component.ts index e98c7d74e..57c29e649 100644 --- a/webapp/src/app/pages/components/knowledge-graph/rating/rating.component.ts +++ b/webapp/src/app/pages/components/knowledge-graph/rating/rating.component.ts @@ -44,7 +44,7 @@ export class RatingComponent { ); } - @Input() element: ArticleElementModel | VideoElementModel; + @Input() element!: ArticleElementModel | VideoElementModel; @Input() notUnderstoodConcepts: any[]; @Output() onClick: EventEmitter = new EventEmitter(); @Input() currentMaterial?: Material; @@ -52,6 +52,7 @@ export class RatingComponent { isLiked = false; isDisliked = false; selectedConcepts: string[] = []; + selectedConceptCids: string[] = []; loggedInUser: User; ngOnInit(): void { @@ -80,10 +81,13 @@ export class RatingComponent { ): void { if (!this.isLiked) { op.toggle(event); - } else { - this.likeElement(element, op); + }else{ + // this.likeElement(element, op); + this.rateRecommendedMaterials(Rating.HELPFUL, true); + this.displayInfofulMessage(); } } + public async likeElement( element: ArticleElementModel | VideoElementModel, op: OverlayPanel @@ -115,7 +119,7 @@ export class RatingComponent { ); } } - await this.rateRecommendedMaterials(Rating.HELPFUL); + await this.rateRecommendedMaterials(Rating.HELPFUL, false); this.displaySuccessfulMessage(); } catch (e) { console.error(e); @@ -153,8 +157,14 @@ export class RatingComponent { ); } } - await this.rateRecommendedMaterials(Rating.NOT_HELPFUL); - this.displaySuccessfulMessage(); + + if (!this.isDisliked) { + await this.rateRecommendedMaterials(Rating.NOT_HELPFUL, false); + this.displaySuccessfulMessage(); + } else { + await this.rateRecommendedMaterials(Rating.NOT_HELPFUL, true); + this.displayInfofulMessage(); + } } catch (e) { console.log(e); } @@ -178,18 +188,32 @@ export class RatingComponent { }); } - async rateRecommendedMaterials(rating: Rating): Promise { + onChangeConcept(event, cid: string) { + this.selectedConceptCids.push(cid); + } + + async rateRecommendedMaterials(rating: Rating, reset: boolean): Promise { const data = { - rating: rating, - resourceId: this.element.id, - concepts: this.selectedConcepts, - }; + user_id: this.userid.toString(), + value: rating.toString(), + rid: this.element.rid, + cids: [...new Set(this.selectedConceptCids)], + reset: reset + } + + const result = await this.materialsRecommenderService.rateRecommendedMaterials(data); + if (!data.reset) { + this.isLiked = result.voted === Rating.HELPFUL; + this.isDisliked = result.voted === Rating.NOT_HELPFUL; + } else { + this.isLiked = false; + this.isDisliked = false; + } + this.element.helpful_count = result.helpful_count; + this.element.not_helpful_count = result.not_helpful_count; + } - const result = - await this.materialsRecommenderService.rateRecommendedMaterials(data); - this.isLiked = result.voted === Rating.HELPFUL; - this.isDisliked = result.voted === Rating.NOT_HELPFUL; - this.element.helpful_counter = result.helpful_count; - this.element.not_helpful_counter = result.not_helpful_count; + displayInfofulMessage(): void { + this.messageService.add({key: 'rating', severity: 'info', summary: '', detail: 'You undo your feedback!'}); } } diff --git a/webapp/src/app/pages/components/knowledge-graph/result-view/result-view.component.css b/webapp/src/app/pages/components/knowledge-graph/result-view/result-view.component.css index e69de29bb..d2a359050 100644 --- a/webapp/src/app/pages/components/knowledge-graph/result-view/result-view.component.css +++ b/webapp/src/app/pages/components/knowledge-graph/result-view/result-view.component.css @@ -0,0 +1,87 @@ + +:host ::ng-deep #cro_sorting .p-panel-title { + color: #0277BD; + font-size: 14px; +} + +:host ::ng-deep #cro_sorting .p-panel { + border: 2px solid #cecece; +} + +:host ::ng-deep .scroll_panel_list { + width: 100%; + height: 55rem; +} + +/* :host ::ng-deep .p-tabview-panels { + position: fixed; +} */ + +:host ::ng-deep .p-paginator { + height: 3.7rem; +} + +/* from TagMenu: move bookmarks to left corner */ +:host ::ng-deep .p-tabview-nav-content ul.p-tabview-nav > li:nth-child(3) { + margin-left: auto; +} + +.left_panel_interaction { + pointer-events: none; + opacity: 0.3; +} + +/* search videos_articles */ +.p-float-label input { + width: 30rem; /* Set to desired width */ + padding-right: 2rem; +} +.p-float-label { + position: relative; +} +.p-float-label input { + padding-right: 2rem; /* Adjust padding to fit the icon */ +} +.search-icon { + position: absolute; + right: 10px; + top: 50%; + transform: translateY(-50%); + pointer-events: none; + color: gray; +} + +/* change checkbox color */ +:host ::ng-deep .p-checkbox .p-checkbox-box { + border: 2px solid #0288d1; +} +:host ::ng-deep .p-checkbox-box:hover { + border-color: #0288d1; +} +:host ::ng-deep .p-checkbox-box.p-highlight { + background: #0288d1; + border-color: #0288d1; +} + +:host ::ng-deep #result_concepts_modified .p-multiSelect { + width: 16rem; +} + +:host ::ng-deep .p-multiSelect .p-multiselect-panel { + width: 16rem; +} + +.p-multiselect-panel { + width: 16rem; +} + +/* -- */ + +.close-icon { + position: absolute; + right: 0.5rem; + top: 50%; + transform: translateY(-50%); + cursor: pointer; + color: gray; +} diff --git a/webapp/src/app/pages/components/knowledge-graph/result-view/result-view.component.html b/webapp/src/app/pages/components/knowledge-graph/result-view/result-view.component.html index 76db78da4..ec37c74eb 100644 --- a/webapp/src/app/pages/components/knowledge-graph/result-view/result-view.component.html +++ b/webapp/src/app/pages/components/knowledge-graph/result-view/result-view.component.html @@ -1,118 +1,427 @@ - - -

- Recommendation based on the following concepts: -

- + +
+ + +
+ +
+
+
+
Sort by
+
+ + +
+
+
+ + + + +
+ +
+ +
+ + +
+ +
+
+
+ +
+ +
+ +
+
+
+
+ +
+ +
+ +
+ + +
+ + + - - - - - - - - - - - - - - - - - - - - -

No Recommended videos for the selected Concepts

- - - - - -

No Recommended articles for the selected Concepts

- - - +
+
+
+ + + +
+ +
+
+ No Articles Materials Found +
+
+ +
+
+ +
+

Loading Articles Materials

+
+ + + + + + + +
+ +
+
Content type:
+
+
+ + + +
+
+ + + +
+
+
+ + +
+ +
+ + +
+ + + + + + +
+ +
+ + +
+
+
+ +
+

Loading saved Materials

+
+
+ +
+
+ No videos found +
+
+ No articles found +
+
+
+
+ +
+
+ +
+
+ + + + + + + +
+
+ +
+
+ Total Videos: {{resourcesPagination?.nodes?.videos?.total_items}} +
+ + +
+ +
+
+ Total Articles: {{resourcesPagination?.nodes?.articles?.total_items}} +
+ + +
+ +
+
+ + +
+
+
+
Videos per page:
+ + + +
+ +
+
Articles per page:
+ + + +
+
+
+ +
+
+
+ + + +
+
+
+ + diff --git a/webapp/src/app/pages/components/knowledge-graph/result-view/result-view.component.ts b/webapp/src/app/pages/components/knowledge-graph/result-view/result-view.component.ts index 64fb19778..44267a3f8 100644 --- a/webapp/src/app/pages/components/knowledge-graph/result-view/result-view.component.ts +++ b/webapp/src/app/pages/components/knowledge-graph/result-view/result-view.component.ts @@ -3,6 +3,8 @@ import { Component, Input } from '@angular/core'; import { VideoElementModel } from '../videos/models/video-element.model'; import { ArticleElementModel } from '../articles/models/article-element.model'; import { MenuItem } from 'primeng/api'; +import { ResourcesPagination, UserResourceFilterResult, UserResourceFilterParamsResult, Concept } from 'src/app/models/croForm'; +import { CustomRecommendationOptionService } from 'src/app/services/custom-recommenation-option.service'; import { MaterialsRecommenderService } from 'src/app/services/materials-recommender.service'; import { Subscription } from 'rxjs'; import { SlideConceptsService } from 'src/app/services/slide-concepts.service'; @@ -19,6 +21,13 @@ enum MaterialModels { MODEL_3 = '3', MODEL_4 = '4', } +interface PageEvent { + first: number; + rows: number; + page: number; + pageCount: number; +} + @Component({ selector: 'app-result-view', templateUrl: './result-view.component.html', @@ -41,18 +50,81 @@ export class ResultViewComponent { concepts: any[] = []; recievedVideoResultIsEmpty = true; recievedArticleResultIsEmpty = true; + + @Input() resourcesPagination: ResourcesPagination = null; + @Input() userId: string; + activeIndex: number = 0; + firstPaginator = 0; + rowsPaginator = 10; + + firstVideo: number = 0; + rowsVideo: number = 10; + firstArticle: number = 0; + rowsArticle: number = 10; + options = [ + { label: 10, value: 10 }, + { label: 30, value: 30 }, + { label: 50, value: 50 }, + { label: 100, value: 100 } + ]; + + disabledSortingKeys = false; + orderUpIcon = false; + orderDESC = "Top to bottom" + orderASC = "Bottom to top" + selectedFactorSortingKeys: any = null; + factorSortingKeys = [ + { name: 'Most similar',key: "similarity_score", status: false, orderText: this.orderDESC }, + { name: 'Most recent', key: "creation_date", status: false, orderText: this.orderDESC }, + { name: 'Most viewed', key: "views", status: false, orderText: this.orderDESC }, + { name: 'Most liked', key: "like_count", status: false, orderText: this.orderDESC }, + { name: 'Most rated', key: "user_rating", status: false, orderText: this.orderDESC }, + { name: 'Most saved', key: "saves_count", status: false, orderText: this.orderDESC } + ]; + factorsTabArticle = ["similarity_score", "saves_count", "user_rating"]; + filteringParamsSavedTab = { + user_id: '', + cids: [], + content_type: 'video', + text: '' + } + showSearchIconPinner = false; + filteringResourcesFound: UserResourceFilterResult; + + mainConceptSelected: any | undefined; + midSelected: any | undefined; + sliderNberSelected: any | undefined; + resourcesSaved: UserResourceFilterParamsResult; + filteringParamsSavedTab2 = { + user_id: null, + cids: [], + mids: [], + slider_numbers: [] + } + areResourcesLoaded = false; + ridsUserSaves = []; + conceptsModifiedByUser: Concept[] = []; + conceptModifiedByUserSelected: any | undefined; + + paginatorPage: number = 0; + paginatorRows: number = 10; + rowsPerPageOptions = [10, 30, 50]; + @Input() currentMaterial?: Material; subscriptions: Subscription = new Subscription(); // Manage subscriptions + constructor( private slideConceptservice: SlideConceptsService, private materialsRecommenderService: MaterialsRecommenderService, + private croService: CustomRecommendationOptionService, private store: Store ) { + slideConceptservice.didNotUnderstandConcepts.subscribe((res) => { this.didNotUnderstandConceptsObj = res; this.didNotUnderstandConceptsObj.forEach((el) => { - this.allConceptsObj = this.allConceptsObj.map((e) => - e.id === el.id ? el : e + this.allConceptsObj = this.allConceptsObj?.map((e) => + e.cid === el?.cid ? el : e ); }); }); @@ -60,8 +132,8 @@ export class ResultViewComponent { slideConceptservice.understoodConcepts.subscribe((res) => { this.understoodConceptsObj = res; this.understoodConceptsObj.forEach((el) => { - this.allConceptsObj = this.allConceptsObj.map((e) => - e.id === el.id ? el : e + this.allConceptsObj = this.allConceptsObj?.map((e) => + e.cid === el.cid ? el : e ); }); }); @@ -73,20 +145,16 @@ export class ResultViewComponent { }) ); } - @Input() - public concepts1: any[] = []; - @Input() - public concepts2: any[] = []; - - // @Input() - // public results: any[] = []; - @Input() public results1: any[] = []; - @Input() public results2: any[] = []; - @Input() public results3: any[] = []; - @Input() public results4: any[] = []; + handleOnShowOnHide() { + console.log('OverlayPanel is up and down'); + this.orderUpIcon = this.orderUpIcon === false ? true : false; + } + ngOnInit(): void { console.log('MaterialModels.MODEL_1', MaterialModels.MODEL_1); + this.setLeftPanelMWinWidth(); + this.materialModels = [ { name: 'Model 1', code: MaterialModels.MODEL_1 }, { name: 'Model 2', code: MaterialModels.MODEL_2 }, @@ -102,6 +170,7 @@ export class ResultViewComponent { this.slideConceptservice.updateUnderstoodConcepts( this.conceptFromChipObj ); + this.conceptFromChipObj = null; }, }, ]; @@ -139,14 +208,16 @@ export class ResultViewComponent { }, ]; - this.loadResultForSelectedModel(MaterialModels.MODEL_1); + this.loadResultForSelectedModel(); this.didNotUnderstandConceptsObj = this.slideConceptservice.commonDidNotUnderstandConcepts; this.didNotUnderstandConceptsObj.forEach((el) => { - this.allConceptsObj = this.allConceptsObj.map((e) => - e.id === el.id ? el : e - ); + if (this.allConceptsObj) { + this.allConceptsObj = this.allConceptsObj.map((e) => + e.id === el.id ? el : e + ); + } }); this.understoodConceptsObj = @@ -158,23 +229,16 @@ export class ResultViewComponent { }); } - sortElements( - a: ArticleElementModel | VideoElementModel, - b: ArticleElementModel | VideoElementModel - ): number { - const likeDislikeRatioForA = a.helpful_counter - a.not_helpful_counter; - const likeDislikeRatioForB = b.helpful_counter - b.not_helpful_counter; + ngAfterViewInit() { + } - if (likeDislikeRatioForA === likeDislikeRatioForB) { - return a.similarity_score > b.similarity_score ? -1 : 1; - } else { - return likeDislikeRatioForA > likeDislikeRatioForB ? -1 : 1; - } + ngOnChanges() { + this.loadResultForSelectedModel(); } setChipConcept(concept): void { this.conceptFromChipObj = { - id: concept.id, + // id: concept.id, cid: concept.cid, // deal with cid instead of id 'AMR' name: concept.name, status: concept.status === 'understood' ? 'notUnderstood' : 'understood', @@ -186,81 +250,261 @@ export class ResultViewComponent { // concept // ); } - loadResultForSelectedModel(key): void { - this.results = []; - this.videos = []; - this.articles = []; - - if (key === '1') { - this.results = this.results1; - this.concepts = this.concepts1; - this.allConceptsObj = [...this.concepts1]; - } else if (key === '2') { - this.results = this.results2; - this.concepts = this.concepts1; - this.allConceptsObj = [...this.concepts1]; - } else if (key === '3') { - this.results = this.results3; - this.concepts = this.concepts2; - this.allConceptsObj = [...this.concepts2]; - } else if (key === '4') { - this.results = this.results4; - this.concepts = this.concepts2; - this.allConceptsObj = [...this.concepts2]; - } else { - this.results = this.results1; - } - try { - this.results.forEach((element: any) => { - const e = element.data; - if (e.labels.includes('Video')) { - this.videos.push(e as VideoElementModel); - } else { - this.articles.push(e as ArticleElementModel); - } + loadResultForSelectedModel() { + this.allConceptsObj = this.resourcesPagination?.concepts; + if (this.allConceptsObj) { + this.allConceptsObj.forEach(concept => { + concept["status"] = "notUnderstood" }); + this.concepts = this.allConceptsObj; - this.videos = this.videos.sort((a, b) => this.sortElements(a, b)); - this.articles = this.articles.sort((a, b) => this.sortElements(a, b)); - - if (this.videos) { + if (this.resourcesPagination?.nodes?.videos.total_items > 0) { this.recievedVideoResultIsEmpty = false; this.logUserViewedRecommendedVideos(); } else { this.recievedVideoResultIsEmpty = true; } - if (this.articles) { + if (this.resourcesPagination?.nodes?.articles.total_items > 0) { this.recievedArticleResultIsEmpty = false; } else { this.recievedArticleResultIsEmpty = true; } - } catch (e) { - console.log(e); } - - console.log('ResultViewComponent Videos', this.videos); - console.log('ResultViewComponent Articles', this.articles); } tabChanged(tab) { + this.activeIndex = tab; + this.setLeftPanelMWinWidth(); + this.deactivateDnuInteraction(); + // Pause videos (if any) when changing tabs document.querySelectorAll('iframe').forEach((iframe) => { - const result = iframe.contentWindow.postMessage( - '{"event":"command","func":"pauseVideo","args":""}', - '*' - ); + const result = iframe.contentWindow.postMessage('{"event":"command","func":"pauseVideo","args":""}', '*') }); + if (tab === 0) { this.logUserViewedRecommendedVideos(); } else if (tab === 1) { this.logUserViewedRecommendedArticles(); } + + if (this.activeIndex === 2) { + this.getRidsFromUserSaves(); + this.getConceptsModifiedByUserFromSaves(); + } + this.closeVideoFrame(); + this.deactivateSortingKeys(); + + this.croService.setResultaTabValue(this.activeIndex.toString()); + if (this.activeIndex === 0 || this.activeIndex === 1) { + this.selectedFactorSortingKeys = null; + } + } + + deactivateSortingKeys() { + if (this.activeIndex === 1 ) { + this.disabledSortingKeys = true; + } else { + this.disabledSortingKeys = false; + } + } + + setLeftPanelMWinWidth() { + let ipo_interact = document.getElementById('ipo_interact'); + ipo_interact.style.minWidth = "30rem"; + ipo_interact.style.width = "30rem"; + } + + deactivateDnuInteraction() { + if (this.activeIndex === 2) { + this.filteringParamsSavedTab.user_id = this.userId; + this.getUserResources(this.filteringParamsSavedTab); + } + } + + filteringResourcesSaved() { + this.showSearchIconPinner = this.filteringParamsSavedTab.text.length >= 3 ? true : false; + if (this.filteringParamsSavedTab.text.length >= 3) { + this.getUserResources(this.filteringParamsSavedTab); + } else { + this.filteringResourcesFound = { articles: [], videos: [] }; + } + } + + onContentTypeChange(event: any) { + if (event.value) { + this.getUserResources(this.filteringParamsSavedTab); + } + } + + onInitGetUserResources() { + this.filteringParamsSavedTab.user_id = this.userId; + this.conceptModifiedByUserSelected = [... this.conceptsModifiedByUser] + let cids = this.conceptModifiedByUserSelected.map(concept => concept.cid); + this.filteringParamsSavedTab.cids = cids; + this.getUserResources(this.filteringParamsSavedTab); + } + + getUserResources(params) { + this.areResourcesLoaded = false; + this.filteringResourcesFound = { articles: [], videos: [] }; + this.materialsRecommenderService.filterUserResourcesSavedBy(params) + .subscribe({ + next: (data: UserResourceFilterResult) => { + this.filteringResourcesFound = data; + this.showSearchIconPinner = false; + this.areResourcesLoaded = true; + }, + error: (err) => { + console.log(err); + }, + } + ); + } + + getRidsFromUserSaves() { + this.materialsRecommenderService.getRidsFromUserSaves(this.userId) + .subscribe({ + next: (data: []) => { + this.ridsUserSaves = data; + this.resourcesPagination?.nodes?.videos.content.forEach((video: VideoElementModel) => video.is_bookmarked_fill = this.ridsUserSaves.includes(video.rid) ); + this.resourcesPagination?.nodes?.articles.content.forEach((article: ArticleElementModel) => article.is_bookmarked_fill = this.ridsUserSaves.includes(article.rid) ); + }, + error: (err) => { + console.log(err); + }, + } + ); + } + + getConceptsModifiedByUserFromSaves() { + this.materialsRecommenderService.getConceptsModifiedByUserFromSaves(this.userId) + .subscribe({ + next: (data: Concept[]) => { + this.conceptsModifiedByUser = data; + this.onInitGetUserResources(); + }, + error: (err) => { + console.log(err); + }, + } + ); + } + + onChangeConceptsModifiedByUser(event) { + let cids = event.value.map(concept => concept.cid); + this.filteringParamsSavedTab.cids = cids; + this.getUserResources(this.filteringParamsSavedTab); + } + + closeVideoFrame() { + let button = document.getElementById('backToList'); + if (button) { + button.click(); + } + } + + sortResourcesByKeys(factor, statusRadio: boolean, statusFac: boolean) { + if (statusRadio === true) { + // factor.orderText = factor.orderText === this.orderDESC ? this.orderASC : this.orderDESC; + this.sortResourcesBy(factor); + } else if (statusRadio === false && statusFac === true) { + + factor.orderText = factor.orderText === this.orderDESC ? this.orderASC : this.orderDESC; + this.sortResourcesBy(factor); + } + } + + sortResourcesBy(factor) { + let key = ""; + if (factor.key === "similarity_score") { + key = "similarity_score"; + } else if (factor.key === "creation_date") { + key = "publish_time"; + } else if (factor.key === "views") { + key = "views"; + } else if (factor.key === "user_rating") { + key = "helpful_count"; + } else if (factor.key === "like_count") { + key = "like_count"; + } else if (factor.key === "saves_count") { + key = "saves_count"; + } + + if (factor.orderText === this.orderASC) { + if (this.activeIndex === 0) { + if (key === "publish_time") { + this.resourcesPagination.nodes.videos.content.sort((a, b) => new Date(a[key]).getTime() - new Date(b[key]).getTime()); + } else { + this.resourcesPagination.nodes.videos.content.sort((a, b) => a[key] - b[key]); + } + } else if (this.activeIndex === 1) { + this.resourcesPagination.nodes.articles.content.sort((a, b) => a[key] - b[key]); + } + } else if (factor.orderText === this.orderDESC) { + if (this.activeIndex === 0) { + if (key === "publish_time") { + this.resourcesPagination.nodes.videos.content.sort((a, b) => new Date(b[key]).getTime() - new Date(a[key]).getTime()); + } else { + this.resourcesPagination.nodes.videos.content.sort((a, b) => b[key] - a[key]); + } + } else if (this.activeIndex === 1) { + this.resourcesPagination.nodes.articles.content.sort((a, b) => b[key] - a[key]); + } + } + } + + onPageChangeResourcesPaginator(event: PageEvent, type) { + this.firstPaginator = event.first; + this.rowsPaginator = event.rows; + + const page_number = this.firstPaginator + 1; + const page_size = this.rowsPaginator; + + let reqDataFinal = JSON.parse(localStorage.getItem('resourcesPaginationParams')); + reqDataFinal.rec_params.pagination_params = {page_number: page_number, page_size: page_size}; + localStorage.setItem('resourcesPaginationParams', JSON.stringify(reqDataFinal)); + + this.resourcesPagination = null; + this.materialsRecommenderService + .getRecommendedMaterials(reqDataFinal) + .subscribe({ + next: (result) => { + this.resourcesPagination = result; + }, + complete: () => { + }, + } + ); } + + onPageChangeResourcesPaginatorV2(event) { + const page_number = event.page + 1; + const page_size = event.rows; + + let reqDataFinal = JSON.parse(localStorage.getItem('resourcesPaginationParams')); + reqDataFinal.rec_params.pagination_params = {page_number: page_number, page_size: page_size}; + localStorage.setItem('resourcesPaginationParams', JSON.stringify(reqDataFinal)); + + this.resourcesPagination = null; + this.materialsRecommenderService + .getRecommendedMaterials(reqDataFinal) + .subscribe({ + next: (result) => { + this.resourcesPagination = result; + }, + complete: () => { + }, + } + ); + } + + async logUserViewedRecommendedVideos() { try { const data = { - materialId: this.currentMaterial!._id, + materialId: this.currentMaterial._id, videos: this.videos, materialPage: this.currentPdfPage, }; @@ -285,4 +529,10 @@ export class ResultViewComponent { console.error('Error logging activity:', error); } } + + clearSearch() { + this.filteringParamsSavedTab.text = ''; + this.getUserResources(this.filteringParamsSavedTab); + } + } diff --git a/webapp/src/app/pages/components/knowledge-graph/videos/card-video-list/card-video-list.component.css b/webapp/src/app/pages/components/knowledge-graph/videos/card-video-list/card-video-list.component.css index 55ae03e04..9fcd93a83 100644 --- a/webapp/src/app/pages/components/knowledge-graph/videos/card-video-list/card-video-list.component.css +++ b/webapp/src/app/pages/components/knowledge-graph/videos/card-video-list/card-video-list.component.css @@ -1,6 +1,6 @@ #backToList{ position: absolute; - left: 1.5%; + /* left: 29.5%; */ top: 50%; z-index: 12; } diff --git a/webapp/src/app/pages/components/knowledge-graph/videos/card-video-list/card-video-list.component.html b/webapp/src/app/pages/components/knowledge-graph/videos/card-video-list/card-video-list.component.html index 5319bd398..31010ab07 100644 --- a/webapp/src/app/pages/components/knowledge-graph/videos/card-video-list/card-video-list.component.html +++ b/webapp/src/app/pages/components/knowledge-graph/videos/card-video-list/card-video-list.component.html @@ -1,28 +1,31 @@ -
- -
- + +
+ + +
+ + + -
- - - - - -
+ > + + + \ No newline at end of file diff --git a/webapp/src/app/pages/components/knowledge-graph/videos/card-video-list/card-video-list.component.ts b/webapp/src/app/pages/components/knowledge-graph/videos/card-video-list/card-video-list.component.ts index c843567c0..993d61989 100644 --- a/webapp/src/app/pages/components/knowledge-graph/videos/card-video-list/card-video-list.component.ts +++ b/webapp/src/app/pages/components/knowledge-graph/videos/card-video-list/card-video-list.component.ts @@ -25,6 +25,8 @@ export class CardVideoListComponent { public notUnderstoodConcepts: string[]; public showVideo = false; public video: VideoElementModel; + @Input() userId: string; + @Input() resultTabType: string = ""; @Input() currentMaterial?: Material; @Output() backButtonClicked: EventEmitter = new EventEmitter(); @@ -41,4 +43,8 @@ export class CardVideoListComponent { this.showVideo = !this.showVideo; this.backButtonClicked.emit(); // Notify the parent } + + onResourceRemovedEvent(rid: string) { + this.videoElements = this.videoElements.filter(video => video.rid !== rid); + } } diff --git a/webapp/src/app/pages/components/knowledge-graph/videos/card-video/card-video.component.css b/webapp/src/app/pages/components/knowledge-graph/videos/card-video/card-video.component.css index a7ad93653..bb96a8028 100644 --- a/webapp/src/app/pages/components/knowledge-graph/videos/card-video/card-video.component.css +++ b/webapp/src/app/pages/components/knowledge-graph/videos/card-video/card-video.component.css @@ -33,3 +33,8 @@ overflow: hidden; -webkit-box-orient: vertical; } + +/* info tooltip icon */ +.info_tool_tip { + font-size: 0.8rem; +} \ No newline at end of file diff --git a/webapp/src/app/pages/components/knowledge-graph/videos/card-video/card-video.component.html b/webapp/src/app/pages/components/knowledge-graph/videos/card-video/card-video.component.html index b4c1d6e1a..8f995d74b 100644 --- a/webapp/src/app/pages/components/knowledge-graph/videos/card-video/card-video.component.html +++ b/webapp/src/app/pages/components/knowledge-graph/videos/card-video/card-video.component.html @@ -1,61 +1,113 @@ -
-
- tailwind logo - -
-
-
-

- - Similarity Score: {{ videoElement.similarity_score | percent }} - -
-

- {{ videoElement.views | number }} views - - {{ videoElement.publish_time | dateAgo }} -

-
-

Keyphrases:

-
- {{ concept }} + + +
+
+
+
+
+ tailwind logo +
+
+ +
+
+
+
+

+

+
+
+ +
+
+
+
Similarity score:
+
{{ videoElement.similarity_score | percent }}
+
+ +
+
+
+
+ +
+
+ +
+
+
+ +
+
+
Similarity score:
+
{{ videoElement.similarity_score | percent }}
+
+ +
+
+
+
+ +
+
+ +
+
+
+ +
+
+ +
+

Keyphrases:

+
+
{{concept}}
+
+
+ +
+
Created at: {{ videoElement.publish_time | dateAgo }}
+
Likes: {{ videoElement.like_count | number }}
+
Views: {{ videoElement.views | number }}
+
+
Author: {{ videoElement.channel_title}}
+
+ Description: + + {{ videoElement.description }} + +
-
-

- Description: -

-

- + +
+
+ + +
+
-
+
\ No newline at end of file diff --git a/webapp/src/app/pages/components/knowledge-graph/videos/card-video/card-video.component.ts b/webapp/src/app/pages/components/knowledge-graph/videos/card-video/card-video.component.ts index d999002e5..a7f197686 100644 --- a/webapp/src/app/pages/components/knowledge-graph/videos/card-video/card-video.component.ts +++ b/webapp/src/app/pages/components/knowledge-graph/videos/card-video/card-video.component.ts @@ -1,6 +1,9 @@ +import { MaterialsRecommenderService } from 'src/app/services/materials-recommender.service'; import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'; import { VideoElementModel } from '../models/video-element.model'; import { Material } from 'src/app/models/Material'; +import { MessageService } from 'primeng/api'; + @Component({ selector: 'app-card-video', @@ -8,18 +11,32 @@ import { Material } from 'src/app/models/Material'; styleUrls: ['./card-video.component.css'], }) export class CardVideoComponent { - constructor() {} + constructor( + private messageService: MessageService, + private materialsRecommenderService: MaterialsRecommenderService, + ) {} + DESCRIPTION_MAX_LENGTH = 450; isActive = false; showModal = false; selectedConcepts: string[] = []; @Input() - public videoElement: VideoElementModel; + public videoElement!: VideoElementModel; @Input() public notUnderstoodConcepts: string[]; @Output() onClick: EventEmitter = new EventEmitter(); @Output() onWatchVideo: EventEmitter = new EventEmitter(); + @Input() userId: string; + @Input() TabSaved: boolean = false; + + isDescriptionFullDisplayed = false; + isBookmarkFill = false; + videoDescription = ""; + saveOrRemoveParams = {"user_id": "", "rid": "", "status": false}; + saveOrRemoveStatus = false; + @Input() resultTabType: string = ""; + @Output() resourceRemovedEvent = new EventEmitter(); // take rid @Input() currentMaterial?: Material; ngOnInit(): void {} @@ -30,5 +47,84 @@ export class CardVideoComponent { this.isActive = !this.isActive; this.showModal = !this.showModal; this.onWatchVideo.emit(videoElement); + + this.showLabelMoreDescription(); + } + + ngOnChanges() { + this.saveOrRemoveParams.status = this.videoElement?.is_bookmarked_fill; + this.saveOrRemoveParams.user_id = this.userId; + this.saveOrRemoveParams.rid = this.videoElement?.rid; + } + + showLabelMoreDescription() { + if (this.videoElement?.description.length > 0 ) { + } + } + + showDescriptionFull() { + this.isDescriptionFullDisplayed = this.isDescriptionFullDisplayed === true ? false : true; + } + + addToBookmark() { + this.videoElement.is_bookmarked_fill = this.videoElement?.is_bookmarked_fill === true ? false : true; + this.saveOrRemoveParams.status = this.videoElement?.is_bookmarked_fill; + + this.SaveOrRemoveUserResource(this.saveOrRemoveParams); + this.onResourceRemovedEvent(); + } + + saveOrRemoveBookmark() { + // detail: 'Open your Bookmark List to find this video' + if (this.videoElement.is_bookmarked_fill === true) { + if (this.saveOrRemoveStatus === true) { + this.messageService.add({ key: 'resource_bookmark_video', severity: 'success', summary: '', detail: 'Video saved successfully'}); + } + } else { + if (this.saveOrRemoveStatus === false) { + this.messageService.add({key: 'resource_bookmark_video', severity: 'info', summary: '', detail: 'Video removed from saved'}); + } + } + } + + SaveOrRemoveUserResource(params) { + this.materialsRecommenderService.SaveOrRemoveUserResource(params) + .subscribe({ + next: (data: any) => { + if (data["msg"] == "saved") { + this.saveOrRemoveStatus = true; + this.videoElement.is_bookmarked_fill = true; + } else { + this.saveOrRemoveStatus = false; + this.videoElement.is_bookmarked_fill = false; + } + this.saveOrRemoveBookmark(); + }, + error: (err) => { + console.log(err); + this.saveOrRemoveStatus = false; + this.videoElement.is_bookmarked_fill = false; + }, + } + ); + } + + onResourceRemovedEvent() { + if (this.videoElement.is_bookmarked_fill === false && this.resultTabType === "saved") { // this.isBookmarkFill === false + this.resourceRemovedEvent.emit(this.videoElement.rid); + } + } + + padStringToLength(str) { + const targetLength = 30; + + if (str.length < targetLength) { + // Pad the string with spaces until it reaches the target length + return str.padEnd(targetLength, ' '); + } else { + // Return the string as is if it's already 50 characters or longer + return str; + } } + } diff --git a/webapp/src/app/pages/components/knowledge-graph/videos/mocks/video.mock.ts b/webapp/src/app/pages/components/knowledge-graph/videos/mocks/video.mock.ts index 62f5602ad..307bc558b 100644 --- a/webapp/src/app/pages/components/knowledge-graph/videos/mocks/video.mock.ts +++ b/webapp/src/app/pages/components/knowledge-graph/videos/mocks/video.mock.ts @@ -2,7 +2,7 @@ import {VideoElementModel} from '../models/video-element.model'; export const videoMock: VideoElementModel[] = [ { - id: 'wTZUHiLUY94', + id: 0, // 'wTZUHiLUY94', title: 'Introduction to Recommender Systems', keyphrases: ['Recommender Systems'], thumbnail: 'https://i.ytimg.com/vi/gxXn9LDAdcU/hqdefault.jpg', @@ -11,12 +11,12 @@ export const videoMock: VideoElementModel[] = [ publish_time: '2020-01-01T08:00:05Z', uri: 'https://www.youtube.com/embed/gxXn9LDAdcU?autoplay=1', similarity_score: 0.95, - helpful_counter: 9, - not_helpful_counter: 2, + helpful_count: 9, + not_helpful_count: 2, duration: '4:37', }, { - id: 'n3RKsY2H-NE', + id: 1, // 'n3RKsY2H-NE', title: 'How Recommender Systems Work (Netflix/Amazon)', keyphrases: ['Recommender Systems', 'Machine Learning'], thumbnail: 'https://i.ytimg.com/vi/n3RKsY2H-NE/hqdefault.jpg', @@ -25,12 +25,12 @@ export const videoMock: VideoElementModel[] = [ publish_time: '2020-02-28T13:36:11Z', uri: 'https://www.youtube.com/embed/n3RKsY2H-NE?autoplay=1', similarity_score: 0.90, - helpful_counter: 5, - not_helpful_counter: 0, + helpful_count: 5, + not_helpful_count: 0, duration: '8:18' }, { - id: 'BthUPVwA59s', + id: 2, // 'BthUPVwA59s', title: 'Recommendation systems overview (Building recommendation systems with TensorFlow)', keyphrases: ['Recommender Systems', 'Machine Learning'], thumbnail: 'https://i.ytimg.com/vi/BthUPVwA59s/hqdefault.jpg', @@ -39,12 +39,12 @@ export const videoMock: VideoElementModel[] = [ publish_time: '2021-06-29T16:19:47Z', uri: 'https://www.youtube.com/embed/wTZUHiLUY94?autoplay=1', similarity_score: 0.90, - helpful_counter: 5, - not_helpful_counter: 0, + helpful_count: 5, + not_helpful_count: 0, duration: '8:18' }, { - id: 'U-yq3I9QugQ', + id: 3, // 'U-yq3I9QugQ', title: 'Recommender System in 6 Minutes', keyphrases: ['Recommender Systems', 'Machine Learning'], thumbnail: 'https://i.ytimg.com/vi/U-yq3I9QugQ/hqdefault.jpg', @@ -53,12 +53,12 @@ export const videoMock: VideoElementModel[] = [ publish_time: '2019-09-14T10:59:17Z', uri: 'https://www.youtube.com/embed/U-yq3I9QugQ?autoplay=1', similarity_score: 0.90, - helpful_counter: 4, - not_helpful_counter: 2, + helpful_count: 4, + not_helpful_count: 2, duration: '6:41' }, { - id: '1JRrCEgiyHM', + id: 4, // '1JRrCEgiyHM', title: 'Lecture 41 — Overview of Recommender Systems | Stanford University', keyphrases: ['Recommender Systems', 'Machine Learning'], thumbnail: 'https://i.ytimg.com/vi/1JRrCEgiyHM/hqdefault.jpg', @@ -67,8 +67,8 @@ export const videoMock: VideoElementModel[] = [ publish_time: '2016-04-13T18:48:37Z', uri: 'https://www.youtube.com/embed/1JRrCEgiyHM?autoplay=1', similarity_score: 0.87, - helpful_counter: 2, - not_helpful_counter: 0, + helpful_count: 2, + not_helpful_count: 0, duration: '16:52' } ]; diff --git a/webapp/src/app/pages/components/knowledge-graph/videos/models/video-element.model.ts b/webapp/src/app/pages/components/knowledge-graph/videos/models/video-element.model.ts index 93bc39715..d26697bb2 100644 --- a/webapp/src/app/pages/components/knowledge-graph/videos/models/video-element.model.ts +++ b/webapp/src/app/pages/components/knowledge-graph/videos/models/video-element.model.ts @@ -1,5 +1,5 @@ export interface VideoElementModel { - id: string; + id: number, title: string; thumbnail: string; keyphrases?: string[]; @@ -9,7 +9,13 @@ export interface VideoElementModel { publish_time: string; uri: string; similarity_score: number; - helpful_counter: number; - not_helpful_counter: number; + helpful_count: number; + not_helpful_count: number; duration: string; + bookmarked_count?: number, + like_count?: number, + channel_title?: string, + rid?: string, + updated_at?: string, + is_bookmarked_fill?: boolean } diff --git a/webapp/src/app/pages/components/knowledge-graph/videos/watch-video/watch-video.component.html b/webapp/src/app/pages/components/knowledge-graph/videos/watch-video/watch-video.component.html index 28e50f0c7..8fba35ae2 100644 --- a/webapp/src/app/pages/components/knowledge-graph/videos/watch-video/watch-video.component.html +++ b/webapp/src/app/pages/components/knowledge-graph/videos/watch-video/watch-video.component.html @@ -1,23 +1,27 @@
- -
-
-
-

- + +
+
+
+ + +
+

+
+ +
+

+
+
+

+
+ + + + Similarity Score: {{ video.similarity_score | percent }}
diff --git a/webapp/src/app/pages/components/materials/material/material.component.html b/webapp/src/app/pages/components/materials/material/material.component.html index 52d8a9d62..15943eda4 100644 --- a/webapp/src/app/pages/components/materials/material/material.component.html +++ b/webapp/src/app/pages/components/materials/material/material.component.html @@ -316,4 +316,8 @@ + diff --git a/webapp/src/app/pages/components/materials/material/material.component.ts b/webapp/src/app/pages/components/materials/material/material.component.ts index e0bf639dc..1424363eb 100644 --- a/webapp/src/app/pages/components/materials/material/material.component.ts +++ b/webapp/src/app/pages/components/materials/material/material.component.ts @@ -99,7 +99,7 @@ export class MaterialComponent implements OnInit, OnDestroy, AfterViewChecked { @Output() conceptMapEvent: EventEmitter = new EventEmitter(); @Output() selectedToolEvent: EventEmitter = new EventEmitter(); cmSelected = false; - + showPdfErrorMessage = false; showDialog: boolean = false; showModeratorPrivileges: boolean; @@ -119,6 +119,7 @@ export class MaterialComponent implements OnInit, OnDestroy, AfterViewChecked { > = null; isResetMaterialNotificationsButtonEnabled: boolean; lastMaterialClickedNotificationSettingSubscription: Subscription; + pdfErrorMaterialId: string; constructor( private indicatorService: IndicatorService, public courseService: CourseService, @@ -227,6 +228,7 @@ export class MaterialComponent implements OnInit, OnDestroy, AfterViewChecked { if (this.selectedChannel._id === this.channels['_id']) { this.materials = this.channels['materials']; this.materials.forEach((material) => { + this.showFullMap[material._id] = false; }); } else { @@ -285,6 +287,15 @@ export class MaterialComponent implements OnInit, OnDestroy, AfterViewChecked { this.lastTimeCourseMapperOpened$ = this.store.select( getLastTimeCourseMapperOpened ); + this.pdfViewService.pdfError$.subscribe((materialId) => { + this.pdfErrorMaterialId = materialId; + if (materialId) { + this.showPdfErrorMessage = true; + } + + }); + + } toggleFullMaterialName(materialId: string, event: MouseEvent): void { // event.preventDefault(); @@ -304,7 +315,9 @@ export class MaterialComponent implements OnInit, OnDestroy, AfterViewChecked { if (words.length <= limit) return text; return words.slice(0, limit).join(' ') + '...'; } - + dismissError() { + this.pdfViewService.clearError(); + } getMaterialActivityIndicator(materialId: string) { return combineLatest([ this.allNotifications$, @@ -396,6 +409,7 @@ export class MaterialComponent implements OnInit, OnDestroy, AfterViewChecked { this.intervalService.stopInterval(); // if (this.tabIndex == -1 && this.showModeratorPrivileges) { if (this.tabIndex == -1) { + this.showPdfErrorMessage = false; this.isMaterialSelected = false; this.router.navigate([ @@ -414,6 +428,7 @@ export class MaterialComponent implements OnInit, OnDestroy, AfterViewChecked { this.setShowDialog(); + this.updateSelectedMaterial(); this.materialService .logMaterial(this.courseID, this.selectedMaterial._id) @@ -478,6 +493,7 @@ export class MaterialComponent implements OnInit, OnDestroy, AfterViewChecked { // } if (!this.selectedMaterial) return; switch (this.selectedMaterial.type) { + case 'pdf': this.pdfViewService.setPageNumber(1); let url = @@ -533,6 +549,14 @@ export class MaterialComponent implements OnInit, OnDestroy, AfterViewChecked { // e.index1 = e.index - 1; // this.selectedMaterial = this.materials[e.index1]; + + if( this.pdfErrorMaterialId == this.selectedMaterial._id) { + this.showPdfErrorMessage =true + } + else { +this.showPdfErrorMessage = false; // Reset the error message if the material ID doesn't match + } + if (this.selectedMaterial.type == 'video' && this.selectedMaterial.url) { this.materialService.deleteMaterial(this.selectedMaterial).subscribe({ next: (data) => { @@ -565,7 +589,7 @@ export class MaterialComponent implements OnInit, OnDestroy, AfterViewChecked { }, }); } else if ( - (this.selectedMaterial.type == 'pdf' && this.selectedMaterial.url) || + (this.selectedMaterial.type == 'pdf' && this.selectedMaterial.url && !this.showPdfErrorMessage) || (this.selectedMaterial.type == 'video' && this.selectedMaterial.url == '') ) { this.materialService.deleteFile(this.selectedMaterial).subscribe({ @@ -599,6 +623,8 @@ export class MaterialComponent implements OnInit, OnDestroy, AfterViewChecked { ]); // e.index = 1 + this.pdfViewService.clearError(); + this.showPdfErrorMessage = false; this.showInfo('Material successfully deleted!'); }, error: (err) => { diff --git a/webapp/src/app/pages/components/notifications/notification-dashboard/notification-dashboard.component.html b/webapp/src/app/pages/components/notifications/notification-dashboard/notification-dashboard.component.html index 729373bcc..189e81de0 100644 --- a/webapp/src/app/pages/components/notifications/notification-dashboard/notification-dashboard.component.html +++ b/webapp/src/app/pages/components/notifications/notification-dashboard/notification-dashboard.component.html @@ -69,13 +69,12 @@ [formControlName]="newerNotification._id" [binary]="true" > - + (notificationClicked)=" + onNotificationClicked(newerNotification) + " + >
@@ -101,13 +100,16 @@ [formControlName]="earlierNotification._id" [binary]="true" > - + + + + [notification]="earlierNotification" + (notificationClicked)=" + onNotificationClicked(earlierNotification) + " + > +
diff --git a/webapp/src/app/pages/home/home.component.ts b/webapp/src/app/pages/home/home.component.ts index 2f6b36a19..491d5eeff 100644 --- a/webapp/src/app/pages/home/home.component.ts +++ b/webapp/src/app/pages/home/home.component.ts @@ -80,17 +80,18 @@ export class HomeComponent implements OnInit { getMyCourses() { this.courseService.fetchCourses().subscribe((courses) => { this.courses = courses; - + for (var course of this.courses) { this.Users = []; - console.log(' the retrived course url is: ', course); + //console.log(' the retrived course url is: ', course); this.Users = course.users; - // console.log(course.users[0].role.name) - // let userModerator = this.Users.find( - // (user) => user.role.id === 'moderator' - // ); - - this.buildCardInfo(course.users[0].userId, course); + let userModerator = this.Users.find( + (user) => user.role.name === 'moderator' + ); + if (!userModerator) { + throw new Error(`Moderator not found for course with id ${course._id}`); + } + this.buildCardInfo(userModerator.userId, course); } }); if (this.courseTriggered == false) { diff --git a/webapp/src/app/services/course.service.ts b/webapp/src/app/services/course.service.ts index b34d35011..0c90ce2a2 100644 --- a/webapp/src/app/services/course.service.ts +++ b/webapp/src/app/services/course.service.ts @@ -88,6 +88,7 @@ export class CourseService { return this.http.get(`${this.API_URL}/my-courses`).pipe( tap((courses) => { this.courses = courses; // Store fetched courses + //console.log('Fetched courses:', this.courses); }), catchError((error: HttpErrorResponse) => { // if (error.status === 401) { diff --git a/webapp/src/app/services/custom-recommenation-option.service.ts b/webapp/src/app/services/custom-recommenation-option.service.ts new file mode 100644 index 000000000..9562770e8 --- /dev/null +++ b/webapp/src/app/services/custom-recommenation-option.service.ts @@ -0,0 +1,36 @@ +import { HttpClient, HttpParams } from '@angular/common/http'; +import { Injectable } from '@angular/core'; +import { Router } from '@angular/router'; +import { Observable, lastValueFrom, BehaviorSubject } from 'rxjs'; +import { environment } from 'src/environments/environment'; +import { Neo4jResult } from '../models/croForm'; +import { HTTPOptions } from '../config/config'; + + +@Injectable({ + providedIn: 'root' +}) +export class CustomRecommendationOptionService { + LEAF_URL = `${environment.API_URL}/recommendation/setting`; + + constructor(public router: Router, private http: HttpClient) { } + + private isResultTabSubject = new BehaviorSubject(null); + public isResultTabSelected$ = this.isResultTabSubject.asObservable(); + + public setResultaTabValue(value: string) { + this.isResultTabSubject.next(value); + } + + getConceptsModifiedByUserIdAndMid(mid: string, user_id: string): Observable { + return this.http.get(`${this.LEAF_URL}/get_concepts_modified_by_user_id_and_mid?user_id=${user_id}&mid=${mid}`); + } + + getConceptsModifiedByUserId(user_id: string): Observable { + return this.http.get(`${this.LEAF_URL}/get_concepts_modified_by_user_id?user_id=${user_id}`); + } + + getConceptsByCids(user_id: string, cids: string): Observable { + return this.http.get(`${this.LEAF_URL}/get_concepts_by_cids?user_id=${user_id}&cids=${cids}`); + } +} \ No newline at end of file diff --git a/webapp/src/app/services/materials-recommender.service.ts b/webapp/src/app/services/materials-recommender.service.ts index ebb0f33c6..5827777d5 100644 --- a/webapp/src/app/services/materials-recommender.service.ts +++ b/webapp/src/app/services/materials-recommender.service.ts @@ -3,25 +3,32 @@ import { Injectable } from '@angular/core'; import { Observable, lastValueFrom } from 'rxjs'; import { environment } from 'src/environments/environment'; import { HTTPOptions } from '../config/config'; +import { ResourcesPagination, RatingResource, UserResourceFilterParamsResult, UserResourceFilterResult, Concept } from '../models/croForm'; + + @Injectable({ providedIn: 'root', }) export class MaterialsRecommenderService { - apiURL = environment.API_URL; - recommendedConcepts: any; - recommendedMaterials: any; - recommendedMaterialsRating: any; + apiURL = environment.API_URL + recommendedConcepts: any + recommendedMaterials: any + recommendedMaterialsRating: any + LEAF_URL = `${environment.API_URL}/recommendation`; constructor(private http: HttpClient) {} + getRecommendedConcepts(data: any): Observable { - this.recommendedConcepts = this.http.post( - `${this.apiURL}/courses/${data.courseId}/materials/${data.materialId}/concept-recommendation`, - data, - HTTPOptions - ); - return this.recommendedConcepts; + this.recommendedConcepts = this.http.post(`${this.LEAF_URL}/get_concepts`, data, HTTPOptions); + return this.recommendedConcepts } + + getRecommendedMaterials(data: any): Observable { + this.recommendedMaterials = this.http.post(`${this.LEAF_URL}/get_resources`, data, HTTPOptions); + return this.recommendedMaterials + } + getRecommendedConceptsLog(data: any): Observable { return this.http.post( `${this.apiURL}/courses/${data.courseId}/materials/${data.materialId}/concept-recommendation/log`, @@ -30,18 +37,9 @@ export class MaterialsRecommenderService { ); } - getRecommendedMaterials(data: any): Observable { - this.recommendedMaterials = this.http.post( - `${this.apiURL}/courses/${data.courseId}/materials/${data.materialId}/resource-recommendation`, - data, - HTTPOptions - ); - return this.recommendedMaterials; - } - async rateRecommendedMaterials(data: any): Promise { const recommendedMaterialsRating$ = this.http.post( - `${this.apiURL}/knowledge-graph/rating`, + `${this.LEAF_URL}/rating_resource`, data, HTTPOptions ); @@ -53,7 +51,7 @@ export class MaterialsRecommenderService { logWikiArticleView(data: any): Observable { return this.http.post( - `${this.apiURL}/materials/${data.materialId}/recommended-articles/${data.title}/log`, + `${this.apiURL}/materials/${data.materialId}/recommended-articles/log`, data, HTTPOptions ); @@ -61,14 +59,14 @@ export class MaterialsRecommenderService { logExpandAbstract(data: any): Observable { return this.http.post( - `${this.apiURL}/materials/${data.materialId}/recommended-article/${data.title}/abstract/log-expand`, + `${this.apiURL}/materials/${data.materialId}/recommended-article/abstract/log-expand`, data, HTTPOptions ); } logCollapseAbstract(data: any): Observable { return this.http.post( - `${this.apiURL}/materials/${data.materialId}/recommended-article/${data.title}/abstract/log-collapse`, + `${this.apiURL}/materials/${data.materialId}/recommended-article/abstract/log-collapse`, data, HTTPOptions ); @@ -77,7 +75,7 @@ export class MaterialsRecommenderService { logMarkAsHelpfulArticle(data: any): Promise { return lastValueFrom( this.http.post( - `${this.apiURL}/materials/${data.materialId}/recommended-article/${data.title}/mark-helpful`, + `${this.apiURL}/materials/${data.materialId}/recommended-article/mark-helpful`, data, HTTPOptions ) @@ -96,7 +94,7 @@ export class MaterialsRecommenderService { logMarkAsNotHelpfulArticle(data: any): Promise { return lastValueFrom( this.http.post( - `${this.apiURL}/materials/${data.materialId}/recommended-article/${data.title}/mark-not-helpful`, + `${this.apiURL}/materials/${data.materialId}/recommended-article/mark-not-helpful`, data, HTTPOptions ) @@ -114,7 +112,7 @@ export class MaterialsRecommenderService { logUnmarkAsHelpfulArticle(data: any): Promise { return lastValueFrom( this.http.post( - `${this.apiURL}/materials/${data.materialId}/recommended-article/${data.title}/un-mark-helpful`, + `${this.apiURL}/materials/${data.materialId}/recommended-article/un-mark-helpful`, data, HTTPOptions ) @@ -133,7 +131,7 @@ export class MaterialsRecommenderService { logUnmarkAsNotHelpfulArticle(data: any): Promise { return lastValueFrom( this.http.post( - `${this.apiURL}/materials/${data.materialId}/recommended-article/${data.title}/un-mark-not-helpful`, + `${this.apiURL}/materials/${data.materialId}/recommended-article/un-mark-not-helpful`, data, HTTPOptions ) @@ -166,4 +164,29 @@ export class MaterialsRecommenderService { ) ); } -} + + SaveOrRemoveUserResource(data: any): Observable { + return this.http.post(`${this.LEAF_URL}/save_or_remove_resources`, data, { withCredentials: true }); + } + + getConceptsModifiedByUserFromSaves(user_id: string): Observable { + const url = `${this.LEAF_URL}/setting/get_concepts_modified_by_user_from_saves?user_id=${user_id}`; + return this.http.get(url); + } + + filterUserResourcesSavedBy(data: any): Observable { + return this.http.post( + `${this.LEAF_URL}/user_resources/filter`, + data, + HTTPOptions + ); + } + + getRidsFromUserSaves(user_id: string): Observable<[]> { + return this.http.get<[]>( + `${this.LEAF_URL}/user_resources/get_rids_from_user_saves?user_id=${user_id}`, + HTTPOptions + ); + } + +} \ No newline at end of file diff --git a/webapp/src/app/services/notifications.service.ts b/webapp/src/app/services/notifications.service.ts index 16edc9df1..cb29e8553 100644 --- a/webapp/src/app/services/notifications.service.ts +++ b/webapp/src/app/services/notifications.service.ts @@ -38,6 +38,15 @@ import { } from 'src/app/models/Notification'; import * as CourseActions from '../pages/courses/state/course.actions'; +//! TODO: This is hardcoded for the moment, it should be enhanced +const ANNOTATION_OBJECTS = [ + 'annotation', + 'note', + 'question', + 'external-resource', +]; + +const MATERIAL_OBJECTS = ['pdf', 'youtube', 'video']; @Injectable({ providedIn: 'root', }) @@ -82,7 +91,7 @@ export class NotificationsService { let transformedNotifications = notifications.map((notification) => { if ( (notification.annotationAuthorId === user.id && - notification.object === 'annotation') || + ANNOTATION_OBJECTS.includes(notification.object)) || (notification.replyAuthorId === user.id && notification.object === 'reply') ) { @@ -165,7 +174,7 @@ export class NotificationsService { notifications = notifications.map((notification) => { if ( (notification.annotationAuthorId === user.id && - notification.object === 'annotation') || + ANNOTATION_OBJECTS.includes(notification.object)) || (notification.replyAuthorId === user.id && notification.object === 'reply') ) { @@ -509,10 +518,8 @@ export class NotificationsService { let lastWord = notification.activityId.statement.object.definition.type.slice(40); let name = null; - if (lastWord === 'annotation' || lastWord === 'reply') { - name = notification.activityId.statement.object.definition.name[ - 'en-US' - ].substring(lastWord.length); + if (ANNOTATION_OBJECTS.includes(lastWord) || lastWord === 'reply') { + name = notification.activityId.statement.object.definition.name['en-US']; } const extensions = Object.values( notification.activityId.statement.object.definition.extensions @@ -557,7 +564,13 @@ export class NotificationsService { } if ( notification.activityId.statement.object.definition.type === - 'http://www.CourseMapper.de/activityType/material' + 'http://www.CourseMapper.de/activityType/material' || + notification.activityId.statement.object.definition.type === + 'http://www.CourseMapper.de/activityType/pdf' || + notification.activityId.statement.object.definition.type === + 'http://www.CourseMapper.de/activityType/youtube' || + notification.activityId.statement.object.definition.type === + 'http://www.CourseMapper.de/activityType/video' ) { material_id = extensions.id; } else if (extensions.material_id) { @@ -565,16 +578,33 @@ export class NotificationsService { } if ( notification.activityId.statement.object.definition.type === - 'http://www.CourseMapper.de/activityType/annotation' + 'http://www.CourseMapper.de/activityType/annotation' || + notification.activityId.statement.object.definition.type === + 'http://www.CourseMapper.de/activityType/note' || + notification.activityId.statement.object.definition.type === + 'http://www.CourseMapper.de/activityType/question' || + notification.activityId.statement.object.definition.type === + 'http://www.CourseMapper.de/activityType/external-resource' ) { annotation_id = extensions.id; from = resultExtensions?.location?.from ?? null; startPage = resultExtensions?.location?.startPage ?? null; } if ( - resultExtensionFirstKey === - 'http://www.CourseMapper.de/extensions/annotation' && - extensionsFirstKey === 'http://www.CourseMapper.de/extensions/material' + (resultExtensionFirstKey === + 'http://www.CourseMapper.de/extensions/annotation' || + resultExtensionFirstKey === + 'http://www.CourseMapper.de/extensions/note' || + resultExtensionFirstKey === + 'http://www.CourseMapper.de/extensions/question' || + resultExtensionFirstKey === + 'http://www.CourseMapper.de/extensions/external-resource') && + (extensionsFirstKey === + 'http://www.CourseMapper.de/extensions/material' || + extensionsFirstKey === 'http://www.CourseMapper.de/extensions/pdf' || + extensionsFirstKey === + 'http://www.CourseMapper.de/extensions/youtube' || + extensionsFirstKey === 'http://www.CourseMapper.de/extensions/video') ) { annotation_id = resultExtensions.id; from = resultExtensions?.location?.from ?? null; @@ -591,7 +621,13 @@ export class NotificationsService { if ( resultExtensionFirstKey === 'http://www.CourseMapper.de/extensions/reply' && - extensionsFirstKey === 'http://www.CourseMapper.de/extensions/annotation' + (extensionsFirstKey === + 'http://www.CourseMapper.de/extensions/annotation' || + extensionsFirstKey === 'http://www.CourseMapper.de/extensions/note' || + extensionsFirstKey === + 'http://www.CourseMapper.de/extensions/question' || + extensionsFirstKey === + 'http://www.CourseMapper.de/extensions/external-resource') ) { reply_id = resultExtensions.id; from = resultExtensions?.location?.from ?? null; @@ -609,7 +645,12 @@ export class NotificationsService { } if ( extensionsFirstKey === - 'http://www.CourseMapper.de/extensions/annotation' + 'http://www.CourseMapper.de/extensions/annotation' || + extensionsFirstKey === 'http://www.CourseMapper.de/extensions/note' || + extensionsFirstKey === + 'http://www.CourseMapper.de/extensions/question' || + extensionsFirstKey === + 'http://www.CourseMapper.de/extensions/external-resource' ) { annotation_id = extensions.id; from = resultExtensions?.location?.from ?? null; diff --git a/webapp/src/app/services/pdfview.service.ts b/webapp/src/app/services/pdfview.service.ts index 571602ad0..4388ccb3d 100644 --- a/webapp/src/app/services/pdfview.service.ts +++ b/webapp/src/app/services/pdfview.service.ts @@ -15,6 +15,8 @@ export class PdfviewService { totalpages$=this.totalPages.asObservable() private firstPageNumber=new BehaviorSubject(0) firstPageNumber$=this.firstPageNumber.asObservable() + private pdfErrorSubject = new BehaviorSubject(null); // null = no error + pdfError$ = this.pdfErrorSubject.asObservable(); @Output() currentPageNumberEvent: EventEmitter = new EventEmitter(); constructor() { } @@ -35,5 +37,12 @@ export class PdfviewService { setFirstPageNumber(numb:number){ this.firstPageNumber.next(numb) } + emitError(materialId: string) { + this.pdfErrorSubject.next(materialId); + } + + clearError() { + this.pdfErrorSubject.next(null); + } } diff --git a/webapp/src/app/services/topic-channel.service.ts b/webapp/src/app/services/topic-channel.service.ts index 138bd14e7..11137248e 100644 --- a/webapp/src/app/services/topic-channel.service.ts +++ b/webapp/src/app/services/topic-channel.service.ts @@ -49,6 +49,7 @@ export class TopicChannelService { ) .pipe( tap((res) => { + this.topics = res.course?.topics; }) ); diff --git a/webapp/src/app/services/user-concepts.service.ts b/webapp/src/app/services/user-concepts.service.ts index 00df29543..9c43a70d6 100644 --- a/webapp/src/app/services/user-concepts.service.ts +++ b/webapp/src/app/services/user-concepts.service.ts @@ -20,7 +20,7 @@ export class UserConceptsService { `${this.backendEndpointURL}/users/user-concepts/${userID}`, { headers: this.httpHeader } ); - console.log(this.userConcepts) + // console.log(this.userConcepts) return this.userConcepts; } diff --git a/webserver/Dockerfile b/webserver/Dockerfile index 810e8fc42..d8c915ca1 100644 --- a/webserver/Dockerfile +++ b/webserver/Dockerfile @@ -1,9 +1,9 @@ -# syntax=docker/dockerfile:1.5 -FROM node:22.1-slim as base +# syntax=docker/dockerfile:1.15 +FROM node:24.0-slim as base WORKDIR /app -ENV PATH "$PATH:/app/node_modules/.bin" -ENV NODE_ENV production -ENV NPM_CONFIG_UPDATE_NOTIFIER false +ENV PATH="$PATH:/app/node_modules/.bin" +ENV NODE_ENV=production +ENV NPM_CONFIG_UPDATE_NOTIFIER=false FROM base as build diff --git a/webserver/dev.Dockerfile b/webserver/dev.Dockerfile index d1bef44e9..2f45b825e 100644 --- a/webserver/dev.Dockerfile +++ b/webserver/dev.Dockerfile @@ -1,9 +1,9 @@ -# syntax=docker/dockerfile:1.5 -FROM node:22-slim +# syntax=docker/dockerfile:1.15 +FROM node:24-slim WORKDIR /app -ENV PATH "$PATH:/app/node_modules/.bin" -ENV NODE_ENV development +ENV PATH="$PATH:/app/node_modules/.bin" +ENV NODE_ENV=development # No files are added, source directory is expected to be mounted diff --git a/webserver/public/uploads/images/gitignore.txt b/webserver/public/uploads/images/gitignore.txt deleted file mode 100644 index 72e8ffc0d..000000000 --- a/webserver/public/uploads/images/gitignore.txt +++ /dev/null @@ -1 +0,0 @@ -* diff --git a/webserver/public/uploads/pdfs/gitignore.txt b/webserver/public/uploads/pdfs/gitignore.txt deleted file mode 100644 index 72e8ffc0d..000000000 --- a/webserver/public/uploads/pdfs/gitignore.txt +++ /dev/null @@ -1 +0,0 @@ -* diff --git a/webserver/public/uploads/videos/gitignore.txt b/webserver/public/uploads/videos/gitignore.txt deleted file mode 100644 index 72e8ffc0d..000000000 --- a/webserver/public/uploads/videos/gitignore.txt +++ /dev/null @@ -1 +0,0 @@ -* diff --git a/webserver/src/activity-logger/generator/annotation-comment/annotation-generator.js b/webserver/src/activity-logger/generator/annotation-comment/annotation-generator.js index 6ac1025d8..63ef59ea2 100644 --- a/webserver/src/activity-logger/generator/annotation-comment/annotation-generator.js +++ b/webserver/src/activity-logger/generator/annotation-comment/annotation-generator.js @@ -141,13 +141,14 @@ export const generateCreateAnnotationActivity = (req) => { verb = "added"; verbURI = "http://activitystrea.ms/schema/1.0/add"; } + let formattedType = formatActivityType(req.locals.annotation.type); return { ...metadata, actor: createUser(req), verb: createVerb(verbURI, verb), object: createAnnotationObject(req), - //result: createAnnotationCommentMaterialResultObject(req, "annotation"), // Confision related to annotation and comment, perhaps comment needs to be removed + result: createAnnotationCommentResultObject(req, formattedType), // Confision related to annotation and comment, perhaps comment needs to be removed context: createContext(), }; }; @@ -290,6 +291,7 @@ export const generateEditAnnotationActivity = (req) => { }, }, result: { + //TODO: This needs to be removed, use in the notification the deifinition's extensions extensions: { [`${DOMAIN}/extensions/${formattedType}`]: { content: newAnnotation.content, @@ -303,6 +305,39 @@ export const generateEditAnnotationActivity = (req) => { }; }; +// export const generateAddMentionStatement = (req) => { +// const metadata = createMetadata(); +// let annotation = req.locals.annotation; +// let origin = req.get("origin"); +// return { +// ...metadata, +// actor: createUser(req), +// verb: createVerb("http://id.tincanapi.com/verb/mentioned", "mentioned"), +// object: { +// objectType: config.activity, +// id: `${origin}/activity/course/${annotation.courseId}/topic/${annotation.topicId}/channel/${annotation.channelId}/material/${annotation.materialId}/mention/${req.locals.mentionedUser.userId}`, +// definition: { +// type: `${DOMAIN}/activityType/you`, +// name: { +// [config.language]: req.locals.mentionedUser.name, +// }, +// extensions: { +// [`${DOMAIN}/extensions/annotation`]: { +// id: annotation._id, +// material_id: annotation.materialId, +// channel_id: annotation.channelId, +// topic_id: annotation.topicId, +// course_id: annotation.courseId, +// content: annotation.content, +// }, +// }, +// }, +// }, +// result: createAnnotationCommentResultObject(req, "annotation"), +// context: createContext(), +// }; +// }; +//TODO: Actually the type is user, but the notification section uses you/annotation export const generateAddMentionStatement = (req) => { const metadata = createMetadata(); let annotation = req.locals.annotation; @@ -311,15 +346,17 @@ export const generateAddMentionStatement = (req) => { actor: createUser(req), verb: createVerb("http://id.tincanapi.com/verb/mentioned", "mentioned"), object: { - objectType: config.activity, + objectType: "User", definition: { - type: `${DOMAIN}/activityType/user`, + type: `${DOMAIN}/activityType/you`, name: { - [config.language]: req.locals.mentionedUser.name, + [config.language]: `${annotation.content.slice(0, 50)}${ + annotation.content.length > 50 ? " ..." : "" + }`, }, extensions: { - [`${DOMAIN}/extensions/user`]: { - annotationId: annotation._id, + [`${DOMAIN}/extensions/annotation`]: { + id: annotation._id, material_id: annotation.materialId, channel_id: annotation.channelId, topic_id: annotation.topicId, @@ -329,7 +366,7 @@ export const generateAddMentionStatement = (req) => { }, }, }, - //result: createAnnotationCommentResultObject(req, "annotation"), + result: createAnnotationCommentResultObject(req, "annotation"), context: createContext(), }; }; diff --git a/webserver/src/activity-logger/generator/reply/reply-generator.js b/webserver/src/activity-logger/generator/reply/reply-generator.js index 7759a535e..a9f0a2e0f 100644 --- a/webserver/src/activity-logger/generator/reply/reply-generator.js +++ b/webserver/src/activity-logger/generator/reply/reply-generator.js @@ -16,9 +16,10 @@ import { createCommentObject } from "../comment/comment-utils"; let DOMAIN = "http://www.CourseMapper.de"; // TODO: Hardcoded due to frontend implementation +//TODO: (Shoeb) The username should be anonymized const createUserObject = (req) => { let user = req.locals.user; - let annotation = req.locals.annotation; + let reply = req.locals.reply; let author = req.locals.annotation.author; let origin = req.get("origin"); return { @@ -33,12 +34,19 @@ const createUserObject = (req) => { [`${DOMAIN}/extensions/user`]: { id: author.userId, authorName: author.name, - content: annotation.content, - annotation_id: annotation._id, - material_id: annotation.materialId, - channel_id: annotation.channelId, - topic_id: annotation.topicId, - course_id: annotation.courseId, + // content: annotation.content, + // annotation_id: annotation._id, + // material_id: annotation.materialId, + // channel_id: annotation.channelId, + // topic_id: annotation.topicId, + // course_id: annotation.courseId, + replyId: reply._id, + annotation_id: reply.annotationId, + material_id: reply.materialId, + channel_id: reply.channelId, + topic_id: reply.topicId, + course_id: reply.courseId, + content: reply.content, }, }, }, @@ -64,6 +72,7 @@ export const generateReplyToUserActivity = (req) => { actor: createUser(req), verb: createVerb("http://id.tincanapi.com/verb/replied", "replied"), object: createUserObject(req), + result: createReplyAnnotationResultObject(req), context: createContext(), }; }; @@ -188,6 +197,37 @@ export const generateEditReplyActivity = (req) => { }; }; +// export const getNewMentionCreationStatement = (req) => { +// const metadata = createMetadata(); +// let reply = req.locals.reply; +// return { +// ...metadata, +// actor: createUser(req), +// verb: createVerb(`http://id.tincanapi.com/verb/mentioned`, "mentioned"), +// object: { +// objectType: config.activity, +// definition: { +// type: `${DOMAIN}/activityType/user`, +// name: { +// [config.language]: req.locals.mentionedUser.name, +// }, +// extensions: { +// [`${DOMAIN}/extensions/user`]: { +// replyId: reply._id, +// annotation_id: reply.annotationId, +// material_id: reply.materialId, +// channel_id: reply.channelId, +// topic_id: reply.topicId, +// course_id: reply.courseId, +// content: reply.content, +// }, +// }, +// }, +// }, +// //result: createReplyResultObject(req), +// context: createContext(), +// }; +// }; export const getNewMentionCreationStatement = (req) => { const metadata = createMetadata(); let reply = req.locals.reply; @@ -196,15 +236,17 @@ export const getNewMentionCreationStatement = (req) => { actor: createUser(req), verb: createVerb(`http://id.tincanapi.com/verb/mentioned`, "mentioned"), object: { - objectType: config.activity, + objectType: "User", definition: { - type: `${DOMAIN}/activityType/user`, + type: `${DOMAIN}/activityType/you`, name: { - [config.language]: req.locals.mentionedUser.name, + [config.language]: `${reply.content.slice(0, 50)}${ + reply.content.length > 50 ? " ..." : "" + }`, }, extensions: { - [`${DOMAIN}/extensions/user`]: { - replyId: reply._id, + [`${DOMAIN}/extensions/reply`]: { + id: reply._id, annotation_id: reply.annotationId, material_id: reply.materialId, channel_id: reply.channelId, @@ -215,7 +257,7 @@ export const getNewMentionCreationStatement = (req) => { }, }, }, - //result: createReplyResultObject(req), + result: createReplyResultObject(req), context: createContext(), }; }; diff --git a/webserver/src/activity-logger/logger-middlewares/annotation-comment-logger.js b/webserver/src/activity-logger/logger-middlewares/annotation-comment-logger.js index 0a83a5dd4..af93810e7 100644 --- a/webserver/src/activity-logger/logger-middlewares/annotation-comment-logger.js +++ b/webserver/src/activity-logger/logger-middlewares/annotation-comment-logger.js @@ -117,25 +117,37 @@ export const addMentionLogger = async (req, res, next) => { req.locals.category = "mentionedandreplied"; let mentioned = req.locals.isMentionedUsersPresent; if (mentioned > 0) { - const mentionedUsers = req.locals.mentionedUsers; try { - for (const mentionedUser of mentionedUsers) { - req.locals.mentionedUser = mentionedUser; // Add the individual user to req.locals - await activityController.createActivity( - annotationActivityGenerator.generateAddMentionStatement(req) - ); - } - notifications.generateNotificationInfo(req), next(); + req.locals.activity = await activityController.createActivity( + annotationActivityGenerator.generateAddMentionStatement(req), + notifications.generateNotificationInfo(req) + ); } catch (err) { res.status(400).send({ error: "Error saving statement to mongo", err }); } - // try { - // req.locals.activity = await activityController.createActivity( - // annotationActivityGenerator.generateAddMentionStatement(req), - // notifications.generateNotificationInfo(req) - // ); } + next(); }; + +// export const addMentionLogger = async (req, res, next) => { +// req.locals.category = "mentionedandreplied"; +// let mentioned = req.locals.isMentionedUsersPresent; +// if (mentioned > 0) { +// const mentionedUsers = req.locals.mentionedUsers; +// try { +// for (const mentionedUser of mentionedUsers) { +// req.locals.mentionedUser = mentionedUser; // Add the individual user to req.locals +// await activityController.createActivity( +// annotationActivityGenerator.generateAddMentionStatement(req), +// notifications.generateNotificationInfo(req) +// ); +// } +// next(); +// } catch (err) { +// res.status(400).send({ error: "Error saving statement to mongo", err }); +// } +// } +// }; export const hideAnnotationsLogger = async (req, res) => { try { await activityController.createActivity( diff --git a/webserver/src/activity-logger/logger-middlewares/reply-logger.js b/webserver/src/activity-logger/logger-middlewares/reply-logger.js index 29c354607..dc41eb361 100644 --- a/webserver/src/activity-logger/logger-middlewares/reply-logger.js +++ b/webserver/src/activity-logger/logger-middlewares/reply-logger.js @@ -18,7 +18,8 @@ export const createReplyLogger = async (req, res, next) => { export const createReplyToUserLogger = async (req, res, next) => { try { req.locals.activity = await activityController.createActivity( - replyActivityGenerator.generateReplyToUserActivity(req) + replyActivityGenerator.generateReplyToUserActivity(req), + notifications.generateNotificationInfo(req) ); next(); } catch (err) { @@ -79,16 +80,27 @@ export const editReplyLogger = async (req, res, next) => { } }; +// export const newMentionLogger = async (req, res, next) => { +// try { +// let mentionedUsers = req.locals.mentionedUsers; +// for (const mentionedUser of mentionedUsers) { +// req.locals.mentionedUser = mentionedUser; +// await activityController.createActivity( +// replyActivityGenerator.getNewMentionCreationStatement(req), +// notifications.generateNotificationInfo(req) +// ); +// } +// next(); +// } catch (err) { +// res.status(400).send({ error: "Error saving statement to mongo", err }); +// } +// }; export const newMentionLogger = async (req, res, next) => { try { - let mentionedUsers = req.locals.mentionedUsers; - for (const mentionedUser of mentionedUsers) { - req.locals.mentionedUser = mentionedUser; - await activityController.createActivity( - replyActivityGenerator.getNewMentionCreationStatement(req) - ); - } - notifications.generateNotificationInfo(req); + req.locals.activity = await activityController.createActivity( + replyActivityGenerator.getNewMentionCreationStatement(req), + notifications.generateNotificationInfo(req) + ); next(); } catch (err) { res.status(400).send({ error: "Error saving statement to mongo", err }); diff --git a/webserver/src/controllers/channel.controller.js b/webserver/src/controllers/channel.controller.js index cbcba262e..8daee96a7 100644 --- a/webserver/src/controllers/channel.controller.js +++ b/webserver/src/controllers/channel.controller.js @@ -48,130 +48,6 @@ export const getChannel = async (req, res) => { "materials", "-__v" ); - /* foundChannel = await BlockingNotifications.aggregate([ - { - $match: { - courseId: ObjectId(courseId), - userId: ObjectId(userId), - }, - }, - { - $project: { - courseId: 0, - isAnnotationNotificationsEnabled: 0, - isReplyAndMentionedNotificationsEnabled: 0, - isCourseUpdateNotificationsEnabled: 0, - userId: 0, - topics: 0, - }, - }, - { - $set: { - channel: { - $first: { - $filter: { - input: "$channels", - cond: { - $eq: ["$$this.channelId", ObjectId(channelId)], - }, - }, - }, - }, - }, - }, - { - $set: { - materials: { - $filter: { - input: "$materials", - cond: { - $eq: ["$$this.channelId", ObjectId(channelId)], - }, - }, - }, - }, - }, - { - $unset: "channels", - }, - { - $lookup: { - from: "channels", - localField: "channel.channelId", - foreignField: "_id", - as: "lookedUpChannel", - }, - }, - { - $set: { - lookedUpChannel: { - $first: "$lookedUpChannel", - }, - }, - }, - { - $lookup: { - from: "materials", - localField: "materials.materialId", - foreignField: "_id", - as: "lookedUpMaterials", - }, - }, - { - $addFields: { - lookUpMaterialIds: { - $map: { - input: "$lookedUpMaterials", - in: "$$this._id", - }, - }, - }, - }, - - { - $addFields: { - materials: { - $map: { - input: "$materials", - in: { - $mergeObjects: [ - "$$this", - { - $arrayElemAt: [ - "$lookedUpMaterials", - { - $indexOfArray: [ - "$lookUpMaterialIds", - "$$this.materialId", - ], - }, - ], - }, - ], - }, - }, - }, - }, - }, - { - $project: { - lookedUpMaterials: 0, - lookUpMaterialIds: 0, - }, - }, - { - $set: { - "lookedUpChannel.materials": "$materials", - }, - }, - { - $replaceRoot: { - newRoot: { - $mergeObjects: ["$channel", "$lookedUpChannel"], - }, - }, - }, - ]); */ if (!foundChannel) { return res.status(404).send({ error: `Channel with id ${channelId} doesn't exist!`, @@ -197,7 +73,6 @@ export const getChannel = async (req, res) => { .status(500) .send({ message: "Error finding notification settings" }); } - return res.status(200).send({ channel: foundChannel, notificationSettings }); }; diff --git a/webserver/src/controllers/course.controller.js b/webserver/src/controllers/course.controller.js index 5a0db8424..7fa1ed290 100644 --- a/webserver/src/controllers/course.controller.js +++ b/webserver/src/controllers/course.controller.js @@ -77,12 +77,16 @@ export const getMyCourses = async (req, res) => { try { user = await User.findById(userId) .populate({ path: "courses", populate: { path: "role" } }) - .populate({ path: "courses", populate: { path: "courseId" } }); + .populate({ path: "courses", populate: { path: "courseId", populate: [ + { path: "users.role" }, ] } }) + + } catch (err) { return res.status(500).send({ message: "Error finding user" }); } user.courses?.forEach((object) => { + const users = object?.courseId?.users || []; let course = { _id: object?.courseId?._id, name: object?.courseId.name, @@ -98,7 +102,13 @@ export const getMyCourses = async (req, res) => { url: object?.courseId.url, }; results.push(course); + // users.forEach(user => { + // console.log("UserId:", user.userId); + // console.log("Role object:", user.role); // This prints the whole populated role object + // }); }); + + //console.log("getMyCourses called with userId: ", results); return res.status(200).send(results); }; @@ -133,6 +143,7 @@ export const getMyCourses = async (req, res) => { * @param {string} req.params.courseId The id of the course */ export const getCourse = async (req, res) => { + console.log("getCourse called with courseId: called"); const courseId = req.params.courseId; const userId = req.userId; //"63387f529dd66f86548d3537" @@ -150,193 +161,7 @@ export const getCourse = async (req, res) => { let foundCourse; try { - /* foundCourse = await Course.aggregate([ - { - $match: { - _id: new ObjectId(courseId), - }, - }, - { - $lookup: { - from: "roles", - localField: "users.role", - foreignField: "_id", - as: "result", - }, - }, - { - $addFields: { - users: { - $map: { - input: "$users", - as: "user", - in: { - userId: "$$user.userId", - _id: "$$user._id", - role: { - $arrayElemAt: [ - "$result", - { - $indexOfArray: ["$result._id", "$$user.role"], - }, - ], - }, - }, - }, - }, - }, - }, - { - $unset: "result", - }, - { - $lookup: { - from: "channels", - localField: "channels", - foreignField: "_id", - as: "result", - }, - }, - { - $lookup: { - from: "followannotations", - let: { - cId: "$_id", - }, - pipeline: [ - { - $match: { - $expr: { - $and: [ - { - $eq: ["$$cId", "$courseId"], - }, - { - $eq: ["$userId", new ObjectId(userId)], - }, - ], - }, - }, - }, - ], - as: "followannotations", - }, - }, - { - $lookup: { - from: "annotations", - localField: "followannotations.annotationId", - foreignField: "_id", - as: "annotations", - }, - }, - { - $addFields: { - mergedObjects: { - $map: { - input: "$annotations", - in: { - $mergeObjects: [ - { - materialType: "$$this.materialType", - content: "$$this.content", - }, - { - $arrayElemAt: [ - "$followannotations", - { - $indexOfArray: [ - "$followannotations.annotationId", - "$$this._id", - ], - }, - ], - }, - ], - }, - }, - }, - }, - }, - { - $unset: "annotations", - }, - { - $unset: "followannotations", - }, - { - $addFields: { - channels: { - $map: { - input: "$result", - as: "channel", - in: { - $mergeObjects: [ - "$$channel", - { - followingAnnotations: { - $filter: { - input: "$mergedObjects", - as: "mergedObj", - cond: { - $eq: ["$$mergedObj.channelId", "$$channel._id"], - }, - }, - }, - }, - ], - }, - }, - }, - }, - }, - { - $unset: "mergedObjects", - }, - { - $unset: "result", - }, - { - $lookup: { - from: "topics", - localField: "topics", - foreignField: "_id", - as: "result", - }, - }, - { - $addFields: { - topics: { - $map: { - input: "$result", - as: "topic", - in: { - $mergeObjects: [ - "$$topic", - { - channels: { - $filter: { - input: "$channels", - as: "channel", - cond: { - $eq: ["$$channel.topicId", "$$topic._id"], - }, - }, - }, - }, - ], - }, - }, - }, - }, - }, - { - $unset: "result", - }, - { - $unset: "channels", - }, - ]); */ + foundCourse = await Course.findById(courseId) .populate("topics", "-__v") .populate({ path: "users", populate: { path: "role" } }) @@ -349,7 +174,9 @@ export const getCourse = async (req, res) => { } catch (err) { return res.status(500).send({ message: "Error finding a course" }); } - + // foundCourse.users.forEach(user => { + // console.log(`UserId: ${user.userId}, Role: ${user.role?.name}`); + // }); let notificationSettings; try { /* notificationSettings = await BlockingNotification.findOne({ @@ -374,35 +201,14 @@ export const getCourse = async (req, res) => { role: currentUser?.role.name || null, // Attach the role of the found user or null if not found }; - + courseWithUserRole.users.forEach(user => { + console.log(`UserId: ${user.userId}, Role: ${user.role?.name}`); + }); return res.status(200).send({ course: courseWithUserRole, notificationSettings: notificationSettings[0], }); - // TODO: Uncomment these code when logger is added - // results = foundCourse.topics.map((topic) => { - // let channels = topic.channels.map((channel) => { - // return { - // _id: channel._id, - // name: channel.name, - // topic_id: channel.topicId, - // course_id: channel.courseId, - // }; - // }); - // return { - // _id: topic._id, - // name: topic.name, - // course_id: topic.courseId, - // channels: channels, - // }; - // }); - // req.locals = { - // response: results, - // course: foundCourse, - // user: foundUser, - // }; - // return next(); }; /** diff --git a/webserver/src/controllers/knowledgeGraph.controller.js b/webserver/src/controllers/knowledgeGraph.controller.js index 583d2eb31..12b97c772 100644 --- a/webserver/src/controllers/knowledgeGraph.controller.js +++ b/webserver/src/controllers/knowledgeGraph.controller.js @@ -512,7 +512,14 @@ export const searchWikipedia = async (req, res) => { try { const conceptNameEncoded = encodeURIComponent(query); const url = `https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch=${conceptNameEncoded}&utf8=&format=json`; - const response = await axios.get(url); + // const response = await axios.get(url); + const response = await axios.get(url, { + headers: { + // Use your app name + version + contact (email or URL) + "User-Agent": "CourseMapper (coursemapper@example.com)" + }, + timeout: 10000 + }); const searchResults = response.data.query.search; // Add the Wikipedia URL to each search result const resultsWithUrls = searchResults.map(result => { diff --git a/webserver/src/controllers/recommendation.controller.js b/webserver/src/controllers/recommendation.controller.js new file mode 100644 index 000000000..b6b822d51 --- /dev/null +++ b/webserver/src/controllers/recommendation.controller.js @@ -0,0 +1,193 @@ +const neo4j = require("../graph/recommendation.neo4j"); +const redis = require("../graph/redis"); +const socketio = require("../socketio"); + + +export const SaveOrRemveUserResources = async (req, res, next) => { + const data = req.body; + try { + const result = await neo4j.userSavesOrRemovesResource(data); + return res.status(200).send(result); + } catch (err) { + console.error('Error saving or removing resource:', err); + return res.status(500).send({ error: err.message }); + } +} + +export const ratingResource = async (req, res, next) => { + const data = req.body; + try { + const result = await neo4j.userRatesResource(data); + return res.status(200).send(result); + } catch (err) { + console.error('Error in userRatesResource:', err); + return res.status(500).send({ error: err.message }); + } +} + + +// recs user_resources + +export const filterUserResourcesSavedBy = async (req, res, next) => { + const data = req.body; + try { + const result = await neo4j.filterUserResourcesSavedBy(data); + return res.status(200).send(result); + } catch (err) { + console.error('Error filtering resources:', err); + return res.status(500).send({ error: err.message }); + } +} + +export const getRidsFromUserResourcesSaved = async (req, res, next) => { + const userId = req.query.user_id; + try { + const result = await neo4j.getRidsFromUserResourcesSaved(userId); + return res.status(200).send(result); + } catch (err) { + console.error('Error retrieving data:', err); + return res.status(500).send({ error: err.message }); + } +} + + +// recs helper_service + +export async function updateConceptModifiedNode(result, userId = null) { + let resultFinal = []; + if (result && result.length > 0) { + let conceptsModified = []; + if (userId) { + conceptsModified = await neo4j.getConceptsModifiedByUserId(userId); + } + + result.forEach(node => { + node.status = false; + + // Update concept weight modified by the user + conceptsModified.forEach(conceptModified => { + if (node.cid === conceptModified.cid) { + node.weight = conceptModified.weight; + } + }); + + resultFinal.push(node); + }); + + // Sort result by the 'name' property + resultFinal.sort((a, b) => a.name.localeCompare(b.name)); + } + return resultFinal; +} + + +export const getConceptsByCids = async (req, res, next) => { + const userId = req.query.user_id; + let cids = req.query.cids; + if ((userId && userId.length > 0) && (cids && cids.length > 0)) { + cids = String(cids).split(","); + try { + const result = await neo4j.getConceptsByCids(userId, cids); + return res.status(200).send({ "records": result }); + } catch (err) { + console.error('Error retrieving data:', err); + return res.status(500).send({ error: err.message }); + } + } else { + return res.status(200).send({ "records": [] }); + } +} + +export const getConceptsModifiedByUserId = async (req, res, next) => { + const userId = req.query.user_id; + try { + const result = await neo4j.getConceptsModifiedByUserId(userId); + return res.status(200).send(result); + } catch (err) { + console.error('Error retrieving data:', err); + return res.status(500).send({ error: err.message }); + } +} + +export const getConceptsModifiedByUserIdAndMid = async (req, res, next) => { + const mid = req.query.mid; + const userId = req.query.user_id; + try { + let result = await neo4j.getConceptsModifiedByMid(mid); + result = await updateConceptModifiedNode(result, userId); + return res.status(200).send({ "records": result }); + } catch (err) { + console.error('Error retrieving data:', err); + return res.status(500).send({ error: err.message }); + } +} + +export const getConceptsModifiedByUserIdAndSlideId = async (req, res, next) => { + const slideId = req.query.slide_id; + const userId = req.query.user_id; + try { + let result = await neo4j.getConceptsModifiedBySlideId(slideId); + result = await updateConceptModifiedNode(result, userId); + return res.status(200).send({ "records": result }); + } catch (err) { + console.error('Error retrieving data:', err); + return res.status(500).send({ error: err.message }); + } +} + +export const getConceptsModifiedByUserFromSaves = async (req, res, next) => { + const userId = req.query.user_id; + try { + const result = await neo4j.getConceptsModifiedByUserFromSaves(userId); + return res.status(200).send(result); + } catch (err) { + console.error('Error retrieving data:', err); + return res.status(500).send({ error: err.message }); + } +} + + + +// set jobs + +export const getConcepts = async (req, res) => { + const body = req.body; + await redis.addJob("concept-recommendation", {"data": body}, undefined, (result) => { + // socketio.getIO().to("recs :").emit("log", result ); + if (res.headersSent) { + return; + } + if (result.error) { + return res.status(500).send({ error: result.error }); + } + return res.status(200).send(result.result); + }); + // socketio.getIO().to("recs :").emit("log", { addJob:result, pipeline:'recs_get_concepts'}); +} + +export const getResources = async (req, res) => { + const body = req.body; + await redis.addJob("resource-recommendation", {"data": body}, undefined, (result) => { + if (res.headersSent) { + return; + } + if (result.error) { + return res.status(500).send({ error: result.error }); + } + return res.status(200).send(result.result); + }); +} + +// export const getResourcesByMainConcepts = async (req, res) => { +// const mid = req.query.mid; +// await redis.addJob("get_resources_by_main_concepts", {"mid": mid}, undefined, (result) => { +// if (res.headersSent) { +// return; +// } +// if (result.error) { +// return res.status(500).send({ error: result.error }); +// } +// return res.status(200).send(result.result); +// }); +// } + diff --git a/webserver/src/graph/recommendation.neo4j.js b/webserver/src/graph/recommendation.neo4j.js new file mode 100644 index 000000000..e21abb61a --- /dev/null +++ b/webserver/src/graph/recommendation.neo4j.js @@ -0,0 +1,407 @@ +// import { graphDb } from "../graph/neo4j" + +const neo4j = require('neo4j-driver'); +const graphDb = {}; + +export async function connect(url, user, password) { + try { + graphDb.driver = neo4j.driver(url, neo4j.auth.basic(user, password), { disableLosslessIntegers: true }); + await graphDb.driver.verifyConnectivity(); + console.log(`Connected to Neo4j`); + } catch (error) { + console.error('Failed to connect to Neo4j', error); + } +} + + +export async function userSavesOrRemovesResource(data) { + /* + User saves or removes Resource(s) + data: { + "user_id": "vhf", + "rid": "2gdsg", + "status": true (create) | false (remove) => (to create or remove a resource saved from the user list) + } + */ + console.log("Saving or Removing from Resource Saved List"); + const result = { msg: "" }; + + if (data.status === true) { + // Save resource + await graphDb.driver.executeQuery( + ` + MATCH (a:User {uid: $user_id}), (b:Resource {rid: $rid}) + MERGE (a)-[r:HAS_SAVED {user_id: $user_id, rid: $rid}]->(b) + `, + { user_id: data.user_id, rid: data.rid } + ); + result.msg = "saved"; + } else { + // Remove resource + await graphDb.driver.executeQuery( + ` + MATCH (a:User {uid: $user_id})-[r:HAS_SAVED {user_id: $user_id, rid: $rid}]->(b:Resource {rid: $rid}) + DELETE r + `, + { user_id: data.user_id, rid: data.rid } + ); + result.msg = "removed"; + } + + // Update resource's saves_count + await graphDb.driver.executeQuery( + ` + MATCH (a:Resource {rid: $rid}) + OPTIONAL MATCH (a)<-[r:HAS_SAVED {rid: $rid}]-() + WITH a, COUNT(r) AS saves_counter + SET a.saves_count = saves_counter + `, + { rid: data.rid } + ); + + console.log("Saving or Removing from Resource Saved List: Done"); + return result; +} + + +export async function userRatesResource(rating) { + /* + User rates Resource(s) + rating: { + "user_id": "43dukl8", + "rid": "bnm565j", + "value": "HELPFUL" | "NOT_HELPFUL", + "reset": true // to undo any rating + } + */ + console.log("User Rates Resource"); + let resetStatus = false; + let result = {}; + + if (rating.reset === true) { + // Reset (Remove Rating) + await graphDb.driver.executeQuery( + ` + MATCH (a:User)-[r:HAS_RATED]->(b:Resource) + WHERE r.user_id = $user_id AND r.rid = $rid AND r.value = $value + DELETE r + `, + { + user_id: rating.user_id, + rid: rating.rid, + value: rating.value, + } + ); + resetStatus = true; + } else if (rating.value !== "HELPFUL") { + // Add or Update Rating (for non-HELPFUL) + await graphDb.driver.executeQuery( + ` + MATCH (a:User {uid: $user_id}), (b:Resource {rid: $rid}) + MERGE (a)-[r:HAS_RATED {user_id: $user_id, rid: $rid}]->(b) + ON CREATE SET r.cids = [], r.value = $value + ON MATCH SET r.value = $value + `, + { + user_id: rating.user_id, + rid: rating.rid, + value: rating.value, + } + ); + } else { + // Add or Update Rating (HELPFUL with merging `cids`) + await graphDb.driver.executeQuery( + ` + MATCH (a:User {uid: $user_id}), (b:Resource {rid: $rid}) + MERGE (a)-[r:HAS_RATED {user_id: $user_id, rid: $rid}]->(b) + ON CREATE SET r.cids = $cids, r.value = $value + ON MATCH SET r.cids = r.cids + [x IN $cids WHERE NOT x IN r.cids], r.value = $value + `, + { + user_id: rating.user_id, + rid: rating.rid, + value: rating.value, + cids: [...new Set(rating.cids || [])], + } + ); + } + + // Update Resource helpful and not helpful counts + const helpfulCount = ( + await graphDb.driver.executeQuery( + ` + MATCH (a:User)-[r:HAS_RATED {value: 'HELPFUL'}]->(b:Resource {rid: $rid}) + RETURN COUNT(r) AS count + `, + { rid: rating.rid } + ) + ).records[0].get("count"); + + const notHelpfulCount = ( + await graphDb.driver.executeQuery( + ` + MATCH (a:User)-[r:HAS_RATED {value: 'NOT_HELPFUL'}]->(b:Resource {rid: $rid}) + RETURN COUNT(r) AS count + `, + { rid: rating.rid } + ) + ).records[0].get("count"); + + const resourceDetails = ( + await graphDb.driver.executeQuery( + ` + MATCH (a:Resource {rid: $rid}) + SET a.helpful_count = $helpfulCount, a.not_helpful_count = $notHelpfulCount + RETURN a.helpful_count AS helpful_count, a.not_helpful_count AS not_helpful_count + `, + { + rid: rating.rid, + helpfulCount: helpfulCount, + notHelpfulCount: notHelpfulCount, + } + ) + ).records[0]; + + result = { + voted: rating.value, + helpful_count: resourceDetails.get("helpful_count"), + not_helpful_count: resourceDetails.get("not_helpful_count"), + reset_status: resetStatus, + }; + + return result; +} + + + + +// recs user_resources db + +export async function getRidsFromUserResourcesSaved(userId) { + /* + Getting rids from User Resources Saved + */ + console.log("Getting rids from User Resources Saved"); + let nodes = []; + const result = await graphDb.driver.executeQuery( + ` + MATCH (b:User)-[r:HAS_SAVED {user_id: $userId}]->(a:Resource) + RETURN a.rid as rid + `, + { userId: userId } + ); + + nodes = result.records.map(record => record.get('rid')); + return nodes; +} + +export async function filterUserResourcesSavedBy(data) { + /* + Getting User Resources Saved + Filtering by: user_id, cids, content_type, text + */ + console.log("Filtering User Resources Saved"); + const result = { articles: [], videos: [] }; + + const query = ` + MATCH (c:Concept_modified)<-[m:HAS_MODIFIED]-(b:User)-[r:HAS_SAVED]->(a:Resource) + WHERE r.user_id = $userId AND $contentType IN LABELS(a) AND + c.cid IN $cids AND ( + toLower(a.text) CONTAINS toLower($searchText) OR + ANY(keyphrase IN a.keyphrases WHERE toLower(keyphrase) CONTAINS toLower($searchText)) + ) + RETURN DISTINCT LABELS(a) as labels, ID(a) as id, a.rid as rid, a.title as title, a.text as text, + a.thumbnail as thumbnail, a.abstract as abstract, a.post_date as post_date, + a.author_image_url as author_image_url, a.author_name as author_name, + a.keyphrases as keyphrases, a.description as description, a.description_full as description_full, + a.publish_time as publish_time, a.uri as uri, a.duration as duration, + COALESCE(toInteger(a.views), 0) AS views, + COALESCE(toFloat(a.similarity_score), 0.0) AS similarity_score, + COALESCE(toInteger(a.helpful_count), 0) AS helpful_count, + COALESCE(toInteger(a.not_helpful_count), 0) AS not_helpful_count, + COALESCE(toInteger(a.bookmarked_count), 0) AS bookmarked_count, + COALESCE(toInteger(a.like_count), 0) AS like_count, + a.channel_title as channel_title, + a.updated_at as updated_at, + true AS is_bookmarked_fill + `; + + const params = { + userId: data.user_id, + searchText: data.text || '', + cids: data.cids || [], + contentType: data.content_type === 'video' ? 'Video' : 'Article', + }; + const queryResult = await graphDb.driver.executeQuery(query, params); + + const resources = queryResult.records.map(record => ({ + labels: record.get('labels'), + id: record.get('id'), + rid: record.get('rid'), + title: record.get('title'), + text: record.get('text'), + thumbnail: record.get('thumbnail'), + abstract: record.get('abstract'), + // post_date: record.get('post_date'), + // author_image_url: record.get('author_image_url'), + // author_name: record.get('author_name'), + keyphrases: record.get('keyphrases'), + description: record.get('description'), + description_full: record.get('description_full'), + publish_time: record.get('publish_time'), + uri: record.get('uri'), + duration: record.get('duration'), + views: record.get('views'), + similarity_score: record.get('similarity_score'), + helpful_count: record.get('helpful_count'), + not_helpful_count: record.get('not_helpful_count'), + bookmarked_count: record.get('bookmarked_count'), + like_count: record.get('like_count'), + channel_title: record.get('channel_title'), + updated_at: record.get('updated_at'), + is_bookmarked_fill: record.get('is_bookmarked_fill'), + })); + + if (resources.length > 0) { + if (data.content_type === 'video') { + result.videos = resources; + } else if (data.content_type === 'article') { + result.articles = resources; + } else { + result.articles = resources.filter(res => res.labels.includes('Article')); + result.videos = resources.filter(res => res.labels.includes('Video')); + } + } + + return result; +} + +// recs helper_service db + +export async function getConceptsByCids(userId, cids) { + let concepts = []; + const query = ` + MATCH (c:Concept) + WHERE c.cid IN $cids + RETURN DISTINCT c.name as name, c.cid as cid, c.weight as weight + `; + const queryResult = await graphDb.driver.executeQuery(query, { cids }); + concepts = queryResult.records.map(record => ({ + name: record.get('name'), + cid: record.get('cid'), + weight: record.get('weight'), + })); + const modifiedConcepts = await getConceptsModifiedByUserId(userId); + + // Update weights if modified by the user + concepts.forEach(concept => { + const modified = modifiedConcepts.find(c => c.cid === concept.cid); + if (modified) { + concept.weight = modified.weight; + } + }); + return concepts; +} + +export async function getConceptsModifiedByUserId(userId) { + let result = []; + const query = ` + MATCH (a:User)-[r:HAS_MODIFIED]->(b:Concept_modified) + WHERE r.user_id = $userId + RETURN DISTINCT r.user_id as user_id, b.cid as cid, r.weight as weight + `; + const queryResult = await graphDb.driver.executeQuery(query, { userId }); + result = queryResult.records.map(record => ({ + user_id: record.get('user_id'), + cid: record.get('cid'), + weight: record.get('weight'), + })); + return result; +} + +export async function getConceptsModifiedByMid(mid) { + let result = []; + // AND c.type = "main_concept" + const query = ` + MATCH (c:Concept) + WHERE c.mid = $mid AND c.type = 'main_concept' + RETURN c.cid as cid, c.name AS name, c.weight as weight, + c.rank as rank, c.mid as mid + ORDER BY c.name + `; + const queryResult = await graphDb.driver.executeQuery(query, { mid }); + result = queryResult.records.map(record => ({ + cid: record.get('cid'), + name: record.get('name'), + weight: record.get('weight'), + rank: record.get('rank'), + mid: record.get('mid'), + })); + return result; +} + +export async function getConceptsModifiedBySlideId(slideId) { + let result = []; + const query = ` + MATCH (s:Slide)-[:CONTAINS]->(c:Concept) + WHERE s.sid = $slideId + RETURN c.cid as cid, c.name AS name, c.weight as weight, + c.rank as rank, c.mid as mid + `; + const queryResult = await graphDb.driver.executeQuery(query, { slideId }); + result = queryResult.records.map(record => ({ + cid: record.get('cid'), + name: record.get('name'), + weight: record.get('weight'), + rank: record.get('rank'), + mid: record.get('mid'), + })); + return result; +} + +export async function getConceptsModifiedByUserFromSaves(userId) { + let result = []; + const query = ` + MATCH (a:User)-[r:HAS_SAVED]->(b:Resource) + -[r2:BASED_ON]->(c:Concept_modified), + (d:Concept) + WHERE r.user_id = $userId AND c.cid = d.cid + RETURN DISTINCT d.cid as cid, d.name as name + `; + const queryResult = await graphDb.driver.executeQuery(query, { userId: userId }); + result = queryResult.records.map(record => ({ + cid: record.get('cid'), + name: record.get('name'), + })); + return result; +} + + +/* +function recordsToObjects(records) { + return records.map((record) => { + const obj = {}; + record.keys.forEach((key, i) => { + obj[key] = record.get(i); + }); + return obj; + }); +} + +export async function getMainConceptsByMid(mid) { + Getting main concepts by mid + console.log("Getting main concepts by mid"); + const result = await graphDb.driver.executeQuery( + ` + MATCH (c:Concept) + WHERE c.mid = $mid AND c.type = "main_concept" + RETURN ID(c) AS id, c.cid AS cid, + c.name AS name, c.type AS type, c.weight AS weight, + c.mid AS mid + ORDER BY c.name + `, + { mid } + ); + return recordsToObjects(result.records); +} +*/ \ No newline at end of file diff --git a/webserver/src/graph/redis.js b/webserver/src/graph/redis.js index e18395134..ff1c24128 100644 --- a/webserver/src/graph/redis.js +++ b/webserver/src/graph/redis.js @@ -5,7 +5,9 @@ const socketio = require("../socketio"); const redis = {} const listeners = {}; -const pipelines = ['concept-map', 'modify-graph', 'expand-material', 'concept-recommendation', 'resource-recommendation']; +const pipelines = ['concept-map', 'modify-graph', 'expand-material', 'concept-recommendation', 'resource-recommendation', + , 'get_resources_by_main_concepts' +]; const jobTimeout = 30; export async function connect(host, port, database, password) { diff --git a/webserver/src/routes/knowledgeGraph.routes.js b/webserver/src/routes/knowledgeGraph.routes.js index a4f4af261..62f8075b5 100644 --- a/webserver/src/routes/knowledgeGraph.routes.js +++ b/webserver/src/routes/knowledgeGraph.routes.js @@ -1,5 +1,6 @@ const { authJwt } = require("../middlewares"); const controller = require("../controllers/knowledgeGraph.controller"); +const recommendationController = require("../controllers/recommendation.controller"); const logger = require("../activity-logger/logger-middlewares/knowledge-graph-logger"); module.exports = function (app) { @@ -91,10 +92,22 @@ module.exports = function (app) { controller.publishConceptMap ); - app.post( - "/api/courses/:courseId/materials/:materialId/concept-recommendation", - [authJwt.verifyToken, authJwt.isEnrolled], - controller.getConcepts + // app.post( + // "/api/courses/:courseId/materials/:materialId/concept-recommendation", + // [authJwt.verifyToken, authJwt.isEnrolled], + // controller.getConcepts + // ); + + // app.post( + // "/api/courses/:courseId/materials/:materialId/resource-recommendation", + // [authJwt.verifyToken, authJwt.isEnrolled], + // controller.getResources + // ); + + app.get( + "/api/wikipedia/search", + [authJwt.verifyToken], + controller.searchWikipedia ); app.post( "/api/courses/:courseId/materials/:materialId/concept-recommendation/log", @@ -123,15 +136,69 @@ module.exports = function (app) { logger.markConceptAsNotUnderstoodLogger ); app.post( - "/api/courses/:courseId/materials/:materialId/resource-recommendation", - [authJwt.verifyToken, authJwt.isEnrolled], - controller.getResources + "/api/recommendation/user_resources/filter", + [authJwt.verifyToken], + recommendationController.filterUserResourcesSavedBy ); app.get( - "/api/wikipedia/search", + "/api/recommendation/user_resources/get_rids_from_user_saves", [authJwt.verifyToken], - controller.searchWikipedia + recommendationController.getRidsFromUserResourcesSaved + ); + + app.get( + "/api/recommendation/setting/get_concepts_by_cids", + [authJwt.verifyToken], + recommendationController.getConceptsByCids + ); + + app.get( + "/api/recommendation/setting/get_concepts_modified_by_user_id", + [authJwt.verifyToken], + recommendationController.getConceptsModifiedByUserId + ); + + app.get( + "/api/recommendation/setting/get_concepts_modified_by_user_id_and_mid", + [authJwt.verifyToken], + recommendationController.getConceptsModifiedByUserIdAndMid + ); + + app.get( + "/api/recommendation/setting/get_concepts_modified_by_user_id_and_slide_id", + [authJwt.verifyToken], + recommendationController.getConceptsModifiedByUserIdAndSlideId + ); + + app.get( + "/api/recommendation/setting/get_concepts_modified_by_user_from_saves", + [authJwt.verifyToken], + recommendationController.getConceptsModifiedByUserFromSaves + ); + + app.post( + "/api/recommendation/save_or_remove_resources", + [authJwt.verifyToken], + recommendationController.SaveOrRemveUserResources + ); + + app.post( + "/api/recommendation/rating_resource", + [authJwt.verifyToken], + recommendationController.ratingResource + ); + + app.post( + "/api/recommendation/get_concepts", + [authJwt.verifyToken], + recommendationController.getConcepts + ); + + app.post( + "/api/recommendation/get_resources", + [authJwt.verifyToken], + recommendationController.getResources ); app.post( @@ -220,13 +287,13 @@ module.exports = function (app) { logger.viewAllRecommendedArticlesLogger ); app.post( - "/api/materials/:materialId/recommended-article/:title/mark-helpful", + "/api/materials/:materialId/recommended-article/mark-helpful", [authJwt.verifyToken], controller.rateArticle, logger.markArticleAsHelpfulLogger ); app.post( - "/api/materials/:materialId/recommended-article/:title/mark-not-helpful", + "/api/materials/:materialId/recommended-article/mark-not-helpful", [authJwt.verifyToken], controller.rateArticle, logger.markArticleAsUnhelpfulLogger @@ -245,13 +312,13 @@ module.exports = function (app) { ); app.post( - "/api/materials/:materialId/recommended-article/:title/un-mark-helpful", + "/api/materials/:materialId/recommended-article/un-mark-helpful", [authJwt.verifyToken], controller.rateArticle, logger.unmarkArticleAsHelpfulLogger ); app.post( - "/api/materials/:materialId/recommended-article/:title/un-mark-not-helpful", + "/api/materials/:materialId/recommended-article/un-mark-not-helpful", [authJwt.verifyToken], controller.rateArticle, logger.unmarkArticleAsUnhelpfulLogger @@ -269,13 +336,13 @@ module.exports = function (app) { logger.unmarkVideoAsUnhelpfulLogger ); app.post( - "/api/materials/:materialId/recommended-article/:title/abstract/log-expand", + "/api/materials/:materialId/recommended-article/abstract/log-expand", [authJwt.verifyToken], controller.expandedArticleAbstract, logger.expandArticleAbstractLogger ); app.post( - "/api/materials/:materialId/recommended-article/:title/abstract/log-collapse", + "/api/materials/:materialId/recommended-article/abstract/log-collapse", [authJwt.verifyToken], controller.collapsedArticleAbstract, logger.collapseArticleAbstractLogger @@ -283,7 +350,7 @@ module.exports = function (app) { // This endpoint is for the recommended Articles part. app.post( - "/api/materials/:materialId/recommended-articles/:title/log", + "/api/materials/:materialId/recommended-articles/log", [authJwt.verifyToken], controller.viewFullWikipediaArticle, logger.viewFullArticleRecommendedArticleLogger diff --git a/webserver/src/server.js b/webserver/src/server.js index e47662404..6715544b2 100644 --- a/webserver/src/server.js +++ b/webserver/src/server.js @@ -81,6 +81,14 @@ neo4j.connect( process.env.NEO4J_PASSWORD ); +const recs = require("./graph/recommendation.neo4j"); +recs.connect( + process.env.NEO4J_URI, + process.env.NEO4J_USER, + process.env.NEO4J_PASSWORD +); + + // Create connection to Redis const redis = require("./graph/redis"); redis.connect(