diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..e4981aa --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,34 @@ +name: Run main.py every 2 minutes + +on: + schedule: + - cron: '*/4 * * * *' # At every 2nd minute of every hour + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout repo content + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11.0' + + - name: Install Python packages + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade rbiparser + + + - name: Execute Python script + run: | + echo "Checking current directory contents:" + ls -al + pwd + + echo "Running rbiparser download -d "./xls":" + rbiparser download -d "./xls" + + diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml new file mode 100644 index 0000000..7504afd --- /dev/null +++ b/.github/workflows/python-app.yml @@ -0,0 +1,32 @@ +name: Run main.py every 2 minutes + +on: + schedule: + - cron: '*/4 * * * *' # At every 2nd minute of every hour + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout repo content + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11.0' + + - name: Install Python packages + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade rbiparser + + + - name: Execute Python script + run: | + echo "Checking current directory contents:" + ls -al + + echo "Running rbiparser download -d "./xls":" + rbiparser download -d "./xls" + diff --git a/rbiparser/__init__.py b/rbiparser/__init__.py index 2199b65..3960182 100644 --- a/rbiparser/__init__.py +++ b/rbiparser/__init__.py @@ -25,6 +25,7 @@ import glob import string + try: from urlparse import urlparse except ImportError: @@ -68,7 +69,6 @@ exclude_words = ["to", "the", "at", "of", "by", "as", "for", "via"] - def get_sheet_urls(url): """Scrapes the RBI page and gets the list of .xlsx sheets.""" r = requests.get(url) @@ -77,7 +77,7 @@ def get_sheet_urls(url): # Extract the urls. s = soup(r.content, "lxml") - links = s.findAll("a", href=re.compile("\.xlsx$")) + links = s.find_all("a", href=re.compile(".\.xls.")) if len(links) < 1: raise Exception("Couldn't find any .xlsx urls") diff --git a/requirement.txt b/requirement.txt new file mode 100644 index 0000000..37b3f7a --- /dev/null +++ b/requirement.txt @@ -0,0 +1,6 @@ +rbiparser +requests +xlrd +beautifulsoup4 +click +lxml