diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..2220425e9 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,38 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/python +{ + "name": "Java 8", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/java:8", + + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + "settings": {}, + "extensions": [ + "github.vscode-github-actions", + "ms-azuretools.vscode-docker", + "streetsidesoftware.code-spell-checker" + ] + } + }, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + "forwardPorts": [8011], + + // Use 'portsAttributes' to set default properties for specific forwarded ports. + // More info: https://containers.dev/implementors/json_reference/#port-attributes + "portsAttributes": { + "8011": { + "label": "Server", + "onAutoForward": "notify" + } + }, + + // Use 'onCreateCommand' to run commands before the container is created. + "onCreateCommand": "sudo apt-get update && sudo apt-get install -y maven graphviz", + + // Use 'postStartCommand' to run commands after the container is started. + "postStartCommand": "chmod 777 build.sh && sh build.sh && chmod 777 getDependencies.sh && sudo sh getDependencies.sh && cp pikes-tintop/target/pikes-tintop-1.0-SNAPSHOT-jar-with-dependencies.jar ./pikes-tintop-1.0-SNAPSHOT-jar-with-dependencies.jar" +} diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 000000000..c2021bddb --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,93 @@ +name: Docker + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +on: + push: + # Publish semver tags as releases. + branches: [master, develop, Docker] + tags: [ 'v*.*.*' ] + +env: + # Use docker.io for Docker Hub if empty + REGISTRY: ghcr.io + # github.repository as / + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + if: github.event_name != 'pull_request' + uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1 + with: + cosign-release: 'v2.1.1' + + # Set up BuildKit Docker container builder to be able to build + # multi-platform images and export cache + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + if: ${{ github.event_name != 'pull_request' }} + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 28a6075cc..1300c9c96 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ target mytests .DS_Store .idea -*.sh dkm-commandlist.txt *.iml */.classpath @@ -10,4 +9,15 @@ dkm-commandlist.txt */.settings *.versionsBackup pikes.ipr -pikes.iws \ No newline at end of file +pikes.iws +utils +parent +fcw +tint +wordnet +ukb +ukb-3.2 +models +*.tgz* +*.tar.gz* +pikes-tintop-1.0-SNAPSHOT-jar-with-dependencies.jar diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..5049e1171 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM maven:3.9-eclipse-temurin-8-focal as builder + +COPY . / +RUN apt-get update && apt-get install -y git +RUN sh ./build.sh +RUN sh ./getDependencies.sh + +FROM openjdk:8 as server + +COPY run.sh ./ +COPY config-pikes.prop ./ +COPY --from=builder pikes-tintop/target/pikes-tintop-1.0-SNAPSHOT-jar-with-dependencies.jar ./pikes-tintop-1.0-SNAPSHOT-jar-with-dependencies.jar +COPY --from=builder wordnet ./wordnet/ +COPY --from=builder models ./models/ +COPY --from=builder ukb ./ukb/ + +RUN apt-get update && apt-get install -y graphviz && chmod -R 777 ukb + +CMD ["sh", "run.sh"] diff --git a/README.md b/README.md new file mode 100644 index 000000000..69a5c5628 --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +# Pikes - Knowledge Extraction Suite + +## Overview +Pikes is a Knowledge Extraction Suite designed to process and analyze text data, extracting valuable insights and information. This suite is developed as part of a collaborative effort, showcasing the power of combining various technologies and methodologies in natural language processing and semantic web. + +## Features +- **Text Analysis and Processing:** Pikes is equipped to handle complex text analysis, leveraging advanced natural language processing techniques. +- **Knowledge Extraction:** The suite excels in extracting structured knowledge from unstructured text data. +- **Semantic Web Integration:** Pikes integrates seamlessly with semantic web technologies, enhancing data interoperability and usability. + +## Modules +The Pikes project consists of several modules, each contributing to its robust functionality: +- `pikes-naflib` +- `pikes-resources` +- `pikes-tintop` +- `pikes-raid` +- `pikes-rdf` +- `pikes-tintop-client` + +## Docker + +If you want to run the image, you can execute the following command: + +```sh +docker run -p 8011:8011 ghcr.io/benediktheinrichs/pikes:Docker +``` + +## Development + +You can run the included dev container to get a full dev environment (use a machine with at least 4 cores). + +## License +Pikes is licensed under the [GNU General Public License (GPL) version 3](http://www.gnu.org/licenses/gpl.txt). + +## More Information +For more details about Pikes and its capabilities, please visit the [official website](https://pikes.fbk.eu/). diff --git a/build.sh b/build.sh new file mode 100755 index 000000000..ec5bc5aef --- /dev/null +++ b/build.sh @@ -0,0 +1,29 @@ +git clone https://github.com/fbk/utils \ + && cd utils \ + && mvn versions:set -DnewVersion=3.2-SNAPSHOT \ + && mvn clean install -DskipTests -Dgpg.skip \ + && cd .. \ + && git clone https://github.com/fbk/parent \ + && cd parent \ + && git checkout develop \ + && mvn versions:set -DnewVersion=2.3-SNAPSHOT \ + && mvn clean install -DskipTests -Dgpg.skip \ + && cd .. \ + && git clone https://github.com/dkmfbk/rdfpro parent/rdfpro \ + && cd parent/rdfpro \ + && git checkout develop \ + && mvn versions:set -DnewVersion=0.7-SNAPSHOT \ + && mvn clean install -DskipTests -Dgpg.skip \ + && cd ../.. \ + && git clone https://github.com/fbk/fcw \ + && cd fcw \ + && grep -rl "http://model.dbpedia-spotlight.org/en" . | xargs sed -i 's/http:\/\/model.dbpedia-spotlight.org\/en/https:\/\/api.dbpedia-spotlight.org\/en/g' \ + && git checkout develop \ + && mvn versions:set -DnewVersion=1.0-SNAPSHOT \ + && mvn clean install -DskipTests -Dgpg.skip \ + && cd .. \ + && git clone https://github.com/dhfbk/tint \ + && cd tint \ + && mvn clean install -DskipTests -Dgpg.skip \ + && cd .. \ + && mvn clean package -DskipTests -Dgpg.skip -Prelease diff --git a/config-pikes.prop b/config-pikes.prop new file mode 100644 index 000000000..02beaef28 --- /dev/null +++ b/config-pikes.prop @@ -0,0 +1,35 @@ +stanford.customAnnotatorClass.dbps.annotator = dbpedia-annotate +stanford.customAnnotatorClass.dbps.address = http://api.dbpedia-spotlight.org/en +stanford.customAnnotatorClass.dbps.min_confidence = 0.0 +stanford.customAnnotatorClass.dbps.first_confidence = 0.5 +stanford.customAnnotatorClass.dbps.extract_types = 0 + +enable_predicate_matrix = 1 +enable_naf_filter = 1 +enable_on_filter = 1 + +max_text_len = 1000000 + +stanford.annotators = tokenize, ssplit, dbps, pos, simple_pos, lemma, ukb, ner, parse, depparse, conll_parse, mst_fake, mate, semafor, dcoref + +stanford.ssplit.newlineIsSentenceBreak = two + +filter.termSenseCompletion = true +filter.entityRemoveOverlaps = true +filter.entitySpanFixing = true +filter.entityAddition = true +filter.entityValueNormalization = true +filter.linkingCompletion = true +filter.linkingFixing = true +filter.corefSpanFixing = true +filter.corefForRoleDependencies = true +filter.srlRemoveWrongRefs = true +filter.srlRemoveUnknownPredicates = true +filter.srlPredicateAddition = true +filter.srlSelfArgFixing = true +filter.srlSenseMapping = false +filter.srlRoleLinking = none +filter.opinionLinking = none + +stanford.parse.maxlen = 50 +stanford.dcoref.maxdist = 5 diff --git a/getDependencies.sh b/getDependencies.sh new file mode 100755 index 000000000..f48602926 --- /dev/null +++ b/getDependencies.sh @@ -0,0 +1,9 @@ +wget https://wordnetcode.princeton.edu/wn3.1.dict.tar.gz +tar -zxvf wn3.1.dict.tar.gz +mv dict wordnet +wget https://git.rwth-aachen.de/coscine/research/pikesdocker/-/archive/main/pikesdocker-main.tar.gz?path=models +tar -zxvf pikesdocker-main.tar.gz?path=models +mv pikesdocker-main-models/models models +wget --no-check-certificate https://ixa2.si.ehu.eus/ukb/ukb_3.2.tgz +tar -zxvf ukb_3.2.tgz +mv ukb-3.2/bin ukb diff --git a/pikes-raid/pom.xml b/pikes-raid/pom.xml index e42651fa1..44e8028e5 100644 --- a/pikes-raid/pom.xml +++ b/pikes-raid/pom.xml @@ -66,12 +66,20 @@ edu.stanford.nlp stanford-corenlp - 3.4 + ${corenlp-version} + + edu.stanford.nlp + stanford-corenlp + models + ${corenlp-version} + runtime + edu.stanford.nlp stanford-corenlp - models + models-english-kbp + ${corenlp-version} runtime diff --git a/pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/APIManager.java b/pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/APIManager.java index a447fa665..6c80050ab 100644 --- a/pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/APIManager.java +++ b/pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/APIManager.java @@ -204,7 +204,7 @@ public double evaluateSentence(String text) { public double evaluateSentence(CoreMap sentence, int startNodeId, ArrayList blockedNodes) { SemanticGraph dependencies = sentence.get(BasicDependenciesAnnotation.class); - String[] dependenciesList = dependencies.toString("list").split("\n"); + String[] dependenciesList = dependencies.toString(SemanticGraph.OutputFormat.LIST).split("\n"); return evaluateSentence(dependenciesList, startNodeId, blockedNodes); } diff --git a/pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/MaxEntTagger.java b/pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/MaxEntTagger.java index 040709c0a..e739ed057 100644 --- a/pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/MaxEntTagger.java +++ b/pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/MaxEntTagger.java @@ -1,7 +1,7 @@ package eu.fbk.dkm.pikes.raid.mdfsa; import edu.stanford.nlp.ling.HasWord; -import edu.stanford.nlp.ling.Sentence; +import edu.stanford.nlp.ling.SentenceUtils; import edu.stanford.nlp.ling.TaggedWord; import edu.stanford.nlp.tagger.maxent.MaxentTagger; @@ -36,7 +36,7 @@ public String tag(String fn) { List> sentences = MaxentTagger.tokenizeText(new BufferedReader(new StringReader(fn))); for (List sentence : sentences) { ArrayList tSentence = (ArrayList) tagger.tagSentence(sentence); - taggedString = taggedString.concat(Sentence.listToString(tSentence, false)); + taggedString = taggedString.concat(SentenceUtils.listToString(tSentence, false)); } return taggedString; } catch(Exception e) { diff --git a/pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/parser/DependenciesBuilder.java b/pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/parser/DependenciesBuilder.java index 1e7220076..cd0dad9ee 100644 --- a/pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/parser/DependenciesBuilder.java +++ b/pikes-raid/src/main/java/eu/fbk/dkm/pikes/raid/mdfsa/parser/DependenciesBuilder.java @@ -1,7 +1,8 @@ package eu.fbk.dkm.pikes.raid.mdfsa.parser; import edu.stanford.nlp.ling.HasWord; -import edu.stanford.nlp.ling.Sentence; +import edu.stanford.nlp.ling.IndexedWord; +import edu.stanford.nlp.ling.SentenceUtils; import edu.stanford.nlp.parser.lexparser.LexicalizedParser; import edu.stanford.nlp.process.DocumentPreprocessor; import edu.stanford.nlp.trees.*; @@ -58,8 +59,8 @@ public void buildDependeciesTree(String text) { DependencyTree dt = new DependencyTree(); for(TypedDependency td: tdl) { - TreeGraphNode dep = td.dep(); - TreeGraphNode gov = td.gov(); + IndexedWord dep = td.dep(); + IndexedWord gov = td.gov(); GrammaticalRelation gr = td.reln(); String depString = gr.toString() + "^^^" + gov.toString() + "^^^" + dep.toString(); //System.out.println(depString); @@ -78,13 +79,13 @@ public ArrayList buildDependeciesTrees(ArrayList texts) for(String text: texts) { System.out.println(textId); String[] sent = text.split(" "); - Tree parse = this.lp.apply(Sentence.toWordList(sent)); + Tree parse = this.lp.apply(SentenceUtils.toWordList(sent)); GrammaticalStructure gs = this.gsf.newGrammaticalStructure(parse); Collection tdl = gs.typedDependenciesCCprocessed(); DependencyTree curDT = new DependencyTree(); for(TypedDependency td: tdl) { - TreeGraphNode dep = td.dep(); - TreeGraphNode gov = td.gov(); + IndexedWord dep = td.dep(); + IndexedWord gov = td.gov(); GrammaticalRelation gr = td.reln(); String depString = gr.toString() + "^^^" + gov.toString() + "^^^" + dep.toString(); curDT.addDependency(depString); diff --git a/pikes-rdf/pom.xml b/pikes-rdf/pom.xml index ab5c0dc98..49fb16ce7 100644 --- a/pikes-rdf/pom.xml +++ b/pikes-rdf/pom.xml @@ -162,7 +162,7 @@ com.spotify.sparkey sparkey - 2.1.2 + 3.2.5 test diff --git a/pikes-resources/pom.xml b/pikes-resources/pom.xml index 2e82431ea..1ca98d072 100644 --- a/pikes-resources/pom.xml +++ b/pikes-resources/pom.xml @@ -25,7 +25,7 @@ javax.xml.bind jaxb-api - 2.2.7 + 2.4.0-b180830.0359 org.jooq @@ -122,13 +122,13 @@ com.fasterxml.jackson.core jackson-databind - 2.3.1 + 2.16.0-rc1 org.apache.commons commons-csv - 1.2 + 1.10.0 @@ -142,21 +142,21 @@ org.apache.tika tika-parsers - 1.12 + 1.28.5 true net.sourceforge.htmlcleaner htmlcleaner - 2.16 + 2.29 true org.unbescape unbescape - 1.1.3.RELEASE + 1.1.6.RELEASE true @@ -165,7 +165,7 @@ org.apache.commons commons-email - 1.4 + 1.5 @@ -183,7 +183,7 @@ org.apache.tika tika-core - 1.12 + 1.28.5 @@ -193,7 +193,7 @@ com.sun.mail javax.mail - 1.5.2 + 1.6.2 @@ -240,21 +240,21 @@ org.apache.tika tika-parsers - 1.12 + 1.28.5 net.sourceforge.htmlcleaner htmlcleaner - 2.16 + 2.29 org.unbescape unbescape - 1.1.3.RELEASE + 1.1.6.RELEASE diff --git a/pikes-tintop-client/pom.xml b/pikes-tintop-client/pom.xml index 2999d3368..5c0ebd30b 100644 --- a/pikes-tintop-client/pom.xml +++ b/pikes-tintop-client/pom.xml @@ -56,7 +56,7 @@ org.apache.httpcomponents httpclient - 4.4 + 4.5.14 eu.fbk.utils diff --git a/pikes-tintop/pom.xml b/pikes-tintop/pom.xml index 00940393a..aaca5d233 100644 --- a/pikes-tintop/pom.xml +++ b/pikes-tintop/pom.xml @@ -47,6 +47,7 @@ edu.stanford.nlp stanford-corenlp + ${corenlp-version} @@ -57,7 +58,14 @@ edu.stanford.nlp stanford-corenlp models - + ${corenlp-version} + runtime + + + edu.stanford.nlp + stanford-corenlp + models-english-kbp + ${corenlp-version} runtime @@ -74,7 +82,7 @@ com.fasterxml.jackson.core jackson-databind - 2.3.1 + 2.16.0-rc1 @@ -84,12 +92,12 @@ org.jdom jdom2 - 2.0.6 + 2.0.6.1 javax.xml.bind jaxb-api - 2.2.7 + 2.4.0-b180830.0359 runtime @@ -105,7 +113,7 @@ org.glassfish.grizzly grizzly-http-server - 2.3.21 + 2.4.4 org.eclipse.rdf4j diff --git a/pom.xml b/pom.xml index fc0e8960a..ffaee251a 100644 --- a/pom.xml +++ b/pom.xml @@ -38,6 +38,15 @@ + + central + Maven Central + default + https://repo1.maven.org/maven2 + + false + + fbk-repo https://raw.githubusercontent.com/fbk/mvn-repo/master/releases @@ -138,7 +147,7 @@ 3.2-SNAPSHOT 0.7-SNAPSHOT 1.0-SNAPSHOT - 3.9.2 + 4.5.5 2.2.4 http://pikes.fbk.eu/ diff --git a/run.sh b/run.sh new file mode 100755 index 000000000..661f6d78a --- /dev/null +++ b/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash +CLASSPATH=pikes-tintop-1.0-SNAPSHOT-jar-with-dependencies.jar +CONF=config-pikes.prop +PORT=8011 +RAM=-Xmx8G +java $RAM -cp $CLASSPATH eu.fbk.dkm.pikes.tintop.server.PipelineServer -p $PORT -c $CONF