Docker Build & Dependency Upgrade

.devcontainer/devcontainer.json

@@ -0,0 +1,38 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+	"name": "Java 8",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/java:8",
+
+	// Configure tool-specific properties.
+	"customizations": {
+		// Configure properties specific to VS Code.
+		"vscode": {
+			"settings": {},
+			"extensions": [
+				"github.vscode-github-actions",
+				"ms-azuretools.vscode-docker",
+				"streetsidesoftware.code-spell-checker"
+			]
+		}
+	},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [8011],
+
+	// Use 'portsAttributes' to set default properties for specific forwarded ports. 
+	// More info: https://containers.dev/implementors/json_reference/#port-attributes
+	"portsAttributes": {
+		"8011": {
+			"label": "Server",
+			"onAutoForward": "notify"
+		}
+	},
+
+	// Use 'onCreateCommand' to run commands before the container is created.
+	"onCreateCommand": "sudo apt-get update && sudo apt-get install -y maven graphviz",
+
+	// Use 'postStartCommand' to run commands after the container is started.
+	"postStartCommand": "chmod 777 build.sh && sh build.sh && chmod 777 getDependencies.sh && sudo sh getDependencies.sh && cp pikes-tintop/target/pikes-tintop-1.0-SNAPSHOT-jar-with-dependencies.jar ./pikes-tintop-1.0-SNAPSHOT-jar-with-dependencies.jar"
+}

.github/workflows/docker-publish.yml

@@ -0,0 +1,93 @@
+name: Docker
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+on:
+  push:
+    # Publish semver tags as releases.
+    branches: [master, develop, Docker]
+    tags: [ 'v*.*.*' ]
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1
+        with:
+          cosign-release: 'v2.1.1'
+
+      # Set up BuildKit Docker container builder to be able to build
+      # multi-platform images and export cache
+      # https://github.com/docker/setup-buildx-action
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Sign the resulting Docker image digest except on PRs.
+      # This will only write to the public Rekor transparency log when the Docker
+      # repository is public to avoid leaking data.  If you would like to publish
+      # transparency data even for private images, pass --force to cosign below.
+      # https://github.com/sigstore/cosign
+      - name: Sign the published Docker image
+        if: ${{ github.event_name != 'pull_request' }}
+        env:
+          # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
+          TAGS: ${{ steps.meta.outputs.tags }}
+          DIGEST: ${{ steps.build-and-push.outputs.digest }}
+        # This step uses the identity token to provision an ephemeral certificate
+        # against the sigstore community Fulcio instance.
+        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}

.gitignore

@@ -2,12 +2,22 @@ target
 mytests
 .DS_Store
 .idea
-*.sh
 dkm-commandlist.txt
 *.iml
 */.classpath
 */.project
 */.settings
 *.versionsBackup
 pikes.ipr
-pikes.iws
+pikes.iws
+utils
+parent
+fcw
+tint
+wordnet
+ukb
+ukb-3.2
+models
+*.tgz*
+*.tar.gz*
+pikes-tintop-1.0-SNAPSHOT-jar-with-dependencies.jar

Dockerfile

@@ -0,0 +1,19 @@
+FROM maven:3.9-eclipse-temurin-8-focal as builder
+
+COPY . /
+RUN apt-get update && apt-get install -y git
+RUN sh ./build.sh
+RUN sh ./getDependencies.sh
+
+FROM openjdk:8 as server
+
+COPY run.sh ./
+COPY config-pikes.prop ./
+COPY --from=builder pikes-tintop/target/pikes-tintop-1.0-SNAPSHOT-jar-with-dependencies.jar ./pikes-tintop-1.0-SNAPSHOT-jar-with-dependencies.jar
+COPY --from=builder wordnet ./wordnet/
+COPY --from=builder models ./models/
+COPY --from=builder ukb ./ukb/
+
+RUN apt-get update && apt-get install -y graphviz && chmod -R 777 ukb
+
+CMD ["sh", "run.sh"]

README.md

@@ -0,0 +1,36 @@
+# Pikes - Knowledge Extraction Suite
+
+## Overview
+Pikes is a Knowledge Extraction Suite designed to process and analyze text data, extracting valuable insights and information. This suite is developed as part of a collaborative effort, showcasing the power of combining various technologies and methodologies in natural language processing and semantic web.
+
+## Features
+- **Text Analysis and Processing:** Pikes is equipped to handle complex text analysis, leveraging advanced natural language processing techniques.
+- **Knowledge Extraction:** The suite excels in extracting structured knowledge from unstructured text data.
+- **Semantic Web Integration:** Pikes integrates seamlessly with semantic web technologies, enhancing data interoperability and usability.
+
+## Modules
+The Pikes project consists of several modules, each contributing to its robust functionality:
+- `pikes-naflib`
+- `pikes-resources`
+- `pikes-tintop`
+- `pikes-raid`
+- `pikes-rdf`
+- `pikes-tintop-client`
+
+## Docker
+
+If you want to run the image, you can execute the following command:
+
+```sh
+docker run -p 8011:8011 ghcr.io/benediktheinrichs/pikes:Docker
+```
+
+## Development
+
+You can run the included dev container to get a full dev environment (use a machine with at least 4 cores).
+
+## License
+Pikes is licensed under the [GNU General Public License (GPL) version 3](http://www.gnu.org/licenses/gpl.txt).
+
+## More Information
+For more details about Pikes and its capabilities, please visit the [official website](https://pikes.fbk.eu/).

build.sh

@@ -0,0 +1,29 @@
+git clone https://github.com/fbk/utils \
+  && cd utils \
+  && mvn versions:set -DnewVersion=3.2-SNAPSHOT \
+  && mvn clean install -DskipTests -Dgpg.skip \
+  && cd .. \
+  && git clone https://github.com/fbk/parent \
+  && cd parent \
+  && git checkout develop \
+  && mvn versions:set -DnewVersion=2.3-SNAPSHOT \
+  && mvn clean install -DskipTests -Dgpg.skip \
+  && cd .. \
+  && git clone https://github.com/dkmfbk/rdfpro parent/rdfpro \
+  && cd parent/rdfpro \
+  && git checkout develop \
+  && mvn versions:set -DnewVersion=0.7-SNAPSHOT \
+  && mvn clean install -DskipTests -Dgpg.skip \
+  && cd ../.. \
+  && git clone https://github.com/fbk/fcw \
+  && cd fcw \
+  && grep -rl "http://model.dbpedia-spotlight.org/en" . | xargs sed -i 's/http:\/\/model.dbpedia-spotlight.org\/en/https:\/\/api.dbpedia-spotlight.org\/en/g' \
+  && git checkout develop \
+  && mvn versions:set -DnewVersion=1.0-SNAPSHOT \
+  && mvn clean install -DskipTests -Dgpg.skip \
+  && cd .. \
+  && git clone https://github.com/dhfbk/tint \
+  && cd tint \
+  && mvn clean install -DskipTests -Dgpg.skip \
+  && cd .. \
+  && mvn clean package -DskipTests -Dgpg.skip -Prelease

config-pikes.prop

@@ -0,0 +1,35 @@
+stanford.customAnnotatorClass.dbps.annotator = dbpedia-annotate
+stanford.customAnnotatorClass.dbps.address = http://api.dbpedia-spotlight.org/en
+stanford.customAnnotatorClass.dbps.min_confidence = 0.0
+stanford.customAnnotatorClass.dbps.first_confidence = 0.5
+stanford.customAnnotatorClass.dbps.extract_types = 0
+
+enable_predicate_matrix = 1
+enable_naf_filter = 1
+enable_on_filter = 1
+
+max_text_len = 1000000
+
+stanford.annotators = tokenize, ssplit, dbps, pos, simple_pos, lemma, ukb, ner, parse, depparse, conll_parse, mst_fake, mate, semafor, dcoref
+
+stanford.ssplit.newlineIsSentenceBreak = two
+
+filter.termSenseCompletion = true
+filter.entityRemoveOverlaps = true
+filter.entitySpanFixing = true
+filter.entityAddition = true
+filter.entityValueNormalization = true
+filter.linkingCompletion = true
+filter.linkingFixing = true
+filter.corefSpanFixing = true
+filter.corefForRoleDependencies = true
+filter.srlRemoveWrongRefs = true
+filter.srlRemoveUnknownPredicates = true
+filter.srlPredicateAddition = true
+filter.srlSelfArgFixing = true
+filter.srlSenseMapping = false
+filter.srlRoleLinking = none
+filter.opinionLinking = none
+
+stanford.parse.maxlen = 50
+stanford.dcoref.maxdist = 5

getDependencies.sh

@@ -0,0 +1,9 @@
+wget https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz
+tar -zxvf wn3.1.dict.tar.gz
+mv dict wordnet
+wget https://git.rwth-aachen.de/coscine/research/pikesdocker/-/archive/main/pikesdocker-main.tar.gz?path=models
+tar -zxvf pikesdocker-main.tar.gz?path=models
+mv pikesdocker-main-models/models models
+wget --no-check-certificate https://ixa2.si.ehu.eus/ukb/ukb_3.2.tgz
+tar -zxvf ukb_3.2.tgz
+mv ukb-3.2/bin ukb

pikes-raid/pom.xml

@@ -66,12 +66,20 @@
         <dependency>
             <groupId>edu.stanford.nlp</groupId>
             <artifactId>stanford-corenlp</artifactId>
-            <version>3.4</version>
+            <version>${corenlp-version}</version>
         </dependency>
+		<dependency>
+			<groupId>edu.stanford.nlp</groupId>
+			<artifactId>stanford-corenlp</artifactId>
+			<classifier>models</classifier>
+            <version>${corenlp-version}</version>
+			<scope>runtime</scope>
+		</dependency>
         <dependency>
             <groupId>edu.stanford.nlp</groupId>
             <artifactId>stanford-corenlp</artifactId>
-            <classifier>models</classifier>
+            <classifier>models-english-kbp</classifier>
+            <version>${corenlp-version}</version>
             <scope>runtime</scope>
         </dependency>
         <dependency>

pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/APIManager.java

@@ -204,7 +204,7 @@ public double evaluateSentence(String text) {
 
   public double evaluateSentence(CoreMap sentence, int startNodeId, ArrayList<Integer> blockedNodes) {
     SemanticGraph dependencies = sentence.get(BasicDependenciesAnnotation.class);
-    String[] dependenciesList = dependencies.toString("list").split("\n");
+    String[] dependenciesList = dependencies.toString(SemanticGraph.OutputFormat.LIST).split("\n");
     return evaluateSentence(dependenciesList, startNodeId, blockedNodes);
   }
 

pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/MaxEntTagger.java

@@ -1,7 +1,7 @@
 package eu.fbk.dkm.pikes.raid.mdfsa;
 
 import edu.stanford.nlp.ling.HasWord;
-import edu.stanford.nlp.ling.Sentence;
+import edu.stanford.nlp.ling.SentenceUtils;
 import edu.stanford.nlp.ling.TaggedWord;
 import edu.stanford.nlp.tagger.maxent.MaxentTagger;
 
@@ -36,7 +36,7 @@ public String tag(String fn) {
 	    List<List<HasWord>> sentences = MaxentTagger.tokenizeText(new BufferedReader(new StringReader(fn)));
 	    for (List<HasWord> sentence : sentences) {
 	    	ArrayList<TaggedWord> tSentence = (ArrayList<TaggedWord>) tagger.tagSentence(sentence);
-	      taggedString = taggedString.concat(Sentence.listToString(tSentence, false));
+	      taggedString = taggedString.concat(SentenceUtils.listToString(tSentence, false));
 	    }
 	    return taggedString; 
 		} catch(Exception e) {

pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/parser/DependenciesBuilder.java

@@ -1,7 +1,8 @@
 package eu.fbk.dkm.pikes.raid.mdfsa.parser;
 
 import edu.stanford.nlp.ling.HasWord;
-import edu.stanford.nlp.ling.Sentence;
+import edu.stanford.nlp.ling.IndexedWord;
+import edu.stanford.nlp.ling.SentenceUtils;
 import edu.stanford.nlp.parser.lexparser.LexicalizedParser;
 import edu.stanford.nlp.process.DocumentPreprocessor;
 import edu.stanford.nlp.trees.*;
@@ -58,8 +59,8 @@ public void buildDependeciesTree(String text) {
 
       DependencyTree dt = new DependencyTree();
       for(TypedDependency td: tdl) {
-        TreeGraphNode dep = td.dep();
-        TreeGraphNode gov = td.gov();
+        IndexedWord dep = td.dep();
+        IndexedWord gov = td.gov();
         GrammaticalRelation gr = td.reln();
         String depString = gr.toString() + "^^^" + gov.toString() + "^^^" + dep.toString();
         //System.out.println(depString);
@@ -78,13 +79,13 @@ public ArrayList<DependencyTree> buildDependeciesTrees(ArrayList<String> texts)
     for(String text: texts) {
      System.out.println(textId);
      String[] sent = text.split(" ");
-      Tree parse = this.lp.apply(Sentence.toWordList(sent));
+      Tree parse = this.lp.apply(SentenceUtils.toWordList(sent));
       GrammaticalStructure gs = this.gsf.newGrammaticalStructure(parse);
       Collection<TypedDependency> tdl = gs.typedDependenciesCCprocessed();
       DependencyTree curDT = new DependencyTree();
       for(TypedDependency td: tdl) {
-        TreeGraphNode dep = td.dep();
-        TreeGraphNode gov = td.gov();
+        IndexedWord dep = td.dep();
+        IndexedWord gov = td.gov();
         GrammaticalRelation gr = td.reln();
         String depString = gr.toString() + "^^^" + gov.toString() + "^^^" + dep.toString();
         curDT.addDependency(depString);

pikes-rdf/pom.xml

@@ -162,7 +162,7 @@
         <dependency>
             <groupId>com.spotify.sparkey</groupId>
             <artifactId>sparkey</artifactId>
-            <version>2.1.2</version>
+            <version>3.2.5</version>
             <scope>test</scope>
         </dependency>
         <dependency>