[pull] master from PaulMcInnis:master

.github/workflows/ci.yml

@@ -4,53 +4,56 @@ name: JobFunnel CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    branches: [master]
 
 jobs:
   build:
-
     runs-on: ubuntu-16.04
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.8
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.8
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-    - name: Install JobFunnel
-      run: |
-        pip install -e .
-        python -m nltk.downloader stopwords
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-    - name: Test with pytest
-      run: |
-        pytest
-    - name: Run CANADA_ENGLISH demo by settings YAML
-      run: |
-        funnel load -s demo/settings.yaml -log-level DEBUG
-    - name: Run an american search by CLI
-      run: |
-        funnel inline -kw Python Data Scientist PHD AI -ps WA -c Seattle -l USA_ENGLISH -log-level DEBUG -csv demo_job_search_results/demo_search.csv -cache demo_job_search_results/cache2 -blf demo_job_search_results/demo_block_list.json -dl demo_job_search_results/demo_duplicates_list.json -log-file demo_job_search_results/log.log -max-listing-days 1
-    - name: Run a FRANCE_FRENCH demo by settings YAML
-      run: |
-        funnel load -s demo/settings_FR.yaml -log-level DEBUG
-    - name: Obtain coverage
-      run: |
-        pytest --cov=jobfunnel --cov-report=xml
-    - name: After Success
-      run: |
-        bash <(curl -s https://codecov.io/bash)
-    - name: Run a Remote jobs only scrape
-      run: |
-        funnel inline -kw Python -ps ON -c Toronto -l CANADA_ENGLISH -remoteness FULLY_REMOTE -p INDEED -log-level DEBUG -csv demo_job_search_results/demo_remote_search.csv -cache demo_job_search_results/cache3 -blf demo_job_search_results/demo_block_list.json -dl demo_job_search_results/demo_duplicates_list.json -log-file demo_job_search_results/log.log -max-listing-days 3
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+      - name: Install JobFunnel
+        run: |
+          pip install -e .
+          python -m nltk.downloader stopwords
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+      - name: Test with pytest
+        run: |
+          pytest
+      - name: Run CANADA_ENGLISH demo by settings YAML
+        run: |
+          funnel load -s demo/settings.yaml -log-level DEBUG
+      - name: Run an american search by CLI
+        run: |
+          funnel inline -kw Python Data Scientist PHD AI -ps WA -c Seattle -l USA_ENGLISH -log-level DEBUG -csv demo_job_search_results/demo_search.csv -cache demo_job_search_results/cache2 -blf demo_job_search_results/demo_block_list.json -dl demo_job_search_results/demo_duplicates_list.json -log-file demo_job_search_results/log.log -max-listing-days 1
+      - name: Run a FRANCE_FRENCH demo by settings YAML
+        run: |
+          funnel load -s demo/settings_FR.yaml -log-level DEBUG
+      - name: Run a GERMANY_GERMAN demo by settings YAML
+        run: |
+          funnel load -s demo/settings_DE.yaml -log-level DEBUG
+      - name: Obtain coverage
+        run: |
+          pytest --cov=jobfunnel --cov-report=xml
+      - name: After Success
+        run: |
+          bash <(curl -s https://codecov.io/bash)
+      - name: Run a Remote jobs only scrape
+        run: |
+          funnel inline -kw Python -ps ON -c Toronto -l CANADA_ENGLISH -remoteness FULLY_REMOTE -p INDEED -log-level DEBUG -csv demo_job_search_results/demo_remote_search.csv -cache demo_job_search_results/cache3 -blf demo_job_search_results/demo_block_list.json -dl demo_job_search_results/demo_duplicates_list.json -log-file demo_job_search_results/log.log -max-listing-days 3
+
 # TODO: modify some job statuses and run with --no-scrape...
 # - './tests/verify_time.sh'   TODO: some way of verifying execution time
-# - './demo/gen_call_graphs.sh'  TODO: some way of showing .dot on GitHub?
+# - './demo/gen_call_graphs.sh'  TODO: some way of showing .dot on GitHub?

.github/workflows/python-publish.yml

@@ -9,23 +9,22 @@ on:
 
 jobs:
   deploy:
-
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload dist/*

.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 24.8.0 # Replace this with the version of Black you want to use
+    hooks:
+      - id: black
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: "v3.1.0" # Specify Prettier version
+    hooks:
+      - id: prettier
+  - repo: https://github.com/pre-commit/mirrors-isort
+    rev: v5.10.1
+    hooks:
+      - id: isort
+        args: ["--profile", "black"]
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8

.prettierignore

@@ -0,0 +1,14 @@
+# Ignore everything by default
+*
+
+# Allow Prettier to format specific file types
+!*.js
+!*.ts
+!*.css
+!*.html
+!*.json
+!*.yaml
+!*.yml
+
+# Allow subdirectories too
+!*/

.prettierrc

@@ -0,0 +1,52 @@
+{
+  "singleQuote": true,
+  "trailingComma": "es5",
+  "printWidth": 80,
+  "tabWidth": 2,
+  "semi": true,
+  "useTabs": false,
+  "bracketSpacing": true,
+  "arrowParens": "always",
+  "proseWrap": "preserve",
+  "overrides": [
+    {
+      "files": "*.json",
+      "options": {
+        "tabWidth": 2
+      }
+    },
+    {
+      "files": "*.md",
+      "options": {
+        "proseWrap": "always",
+        "printWidth": 80
+      }
+    },
+    {
+      "files": "*.html",
+      "options": {
+        "printWidth": 100
+      }
+    },
+    {
+      "files": "*.css",
+      "options": {
+        "printWidth": 80
+      }
+    },
+    {
+      "files": "*.yaml",
+      "options": {
+        "tabWidth": 2,
+        "singleQuote": false
+      }
+    },
+    {
+      "files": "*.yml",
+      "options": {
+        "tabWidth": 2,
+        "singleQuote": false
+      }
+    }
+  ]
+}

MANIFEST.in

@@ -1,5 +1,6 @@
 include jobfunnel/demo/settings.yaml
 include jobfunnel/demo/demo.png
 include jobfunnel/resources/user_agent_list.txt
+include jobfunnel/resources/user_agent_list_mobile.txt
 include readme.md
 include LICENSE

demo/settings.yaml

@@ -11,7 +11,6 @@ log_file: demo_job_search_results/log.log
 
 # Job search configuration
 search:
-
   # Locale settings, one of USA_ENGLISH, CANADA_ENGLISH, CANADA_FRENCH,
   # UK_ENGLISH, FRANCE_FRENCH:
   # This tells JobFunnel where the website we are scraping is located, and
@@ -26,9 +25,9 @@ search:
     - MONSTER
 
   # Region that we are searching for jobs within:
-  province_or_state: "ON"  # NOTE: this is generally 2 characters long.
-  city: "Waterloo"  # NOTE: this is the full city / town name.
-  radius: 25  # km (NOTE: if we were in locale: USA_ENGLISH it's in miles)
+  province_or_state: "ON" # NOTE: this is generally 2 characters long.
+  city: "Waterloo" # NOTE: this is the full city / town name.
+  radius: 25 # km (NOTE: if we were in locale: USA_ENGLISH it's in miles)
 
   # These are the terms you would be typing into the website's search field:
   keywords:
@@ -50,19 +49,18 @@ log_level: INFO
 
 # Delaying algorithm configuration
 delay:
-    # Functions used for delaying algorithm: CONSTANT, LINEAR, SIGMOID
-    algorithm: LINEAR
-    # Maximum delay/upper bound for converging random delay
-    max_duration: 5.0
-    # Minimum delay/lower bound for random delay
-    min_duration: 1.0
-    # Random delay
-    random: False
-    # Converging random delay, only used if 'random' is set to True
-    converging: False
-
+  # Functions used for delaying algorithm: CONSTANT, LINEAR, SIGMOID
+  algorithm: LINEAR
+  # Maximum delay/upper bound for converging random delay
+  max_duration: 5.0
+  # Minimum delay/lower bound for random delay
+  min_duration: 1.0
+  # Random delay
+  random: False
+  # Converging random delay, only used if 'random' is set to True
+  converging: False
 # # Proxy settings
 # proxy:
 #   protocol: https  # NOTE: you can also set to 'http'
 #   ip: "1.1.1.1"
-#   port: 200
+#   port: 200

demo/settings_DE.yaml

@@ -0,0 +1,66 @@
+# This is an example of a feature-complete JobFunnel configuration YAML.
+# Try this out by simply running: "funnel load -s demo/settings.yaml"
+
+# Path where your master CSV, block-lists, and cache data will be stored
+# NOTE: we create any missing directories in these filepaths
+master_csv_file: demo_job_search_results/demo_search.csv
+cache_folder: demo_job_search_results/cache
+block_list_file: demo_job_search_results/demo_block_list.json
+duplicates_list_file: demo_job_search_results/demo_duplicates_list.json
+log_file: demo_job_search_results/log.log
+
+# Job search configuration
+search:
+  # Locale settings, one of USA_ENGLISH, CANADA_ENGLISH, CANADA_FRENCH:
+  # This tells JobFunnel where the website we are scraping is located, and
+  # what language the contents are in.
+  locale: GERMANY_GERMAN
+
+  # Job providers which we will search, one of INDEED, MONSTER, GLASSDOOR:
+  # NOTE: we choose domain via locale (i.e. CANADA_ENGLISH -> www.indeed.ca)
+  # FIXME: we need to add back GLASSDOOR when that's working again.
+  providers:
+    - INDEED
+
+  # Region that we are searching for jobs within:
+  province_or_state: "" # Not used for the german scraper.
+  city: "Berlin" # NOTE: this is the full city / town name.
+  radius: 0 # km (NOTE: if we were in locale: USA_ENGLISH it's in miles)
+
+  # These are the terms you would be typing into the website's search field:
+  keywords:
+    - Python
+    - Senior
+    - AI
+
+  # Don't return any listings older than this:
+  max_listing_days: 10
+
+  # Blocked company names that will never appear in any results:
+  company_block_list:
+    - "EvilCorp"
+
+  # The desired level of work-remoteness (i.e. IN_PERSON, FULLY_REMOTE, ANY,
+  # TEMPORARILY_REMOTE, PARTIALLY_REMOTE)
+  remoteness: ANY
+
+# Logging level options are: critical, error, warning, info, debug, notset
+log_level: INFO
+
+# Delaying algorithm configuration
+delay:
+  # Functions used for delaying algorithm: CONSTANT, LINEAR, SIGMOID
+  algorithm: LINEAR
+  # Maximum delay/upper bound for converging random delay
+  max_duration: 5.0
+  # Minimum delay/lower bound for random delay
+  min_duration: 1.0
+  # Random delay
+  random: True
+  # Converging random delay, only used if 'random' is set to True
+  converging: False
+# # Proxy settings
+# proxy:
+#   protocol: https  # NOTE: you can also set to 'http'
+#   ip: "1.1.1.1"
+#   port: 200