From 4600b6cb990d0f8bafae6f12a2b01f0dfdea0d4d Mon Sep 17 00:00:00 2001 From: Martin Valgur Date: Tue, 13 Apr 2021 15:50:18 +0300 Subject: [PATCH] use external Hatanaka library for RINEX decompression --- README.rst | 14 ----- docs/installation.rst | 23 -------- rakefile | 7 +-- setup.py | 2 +- tecs/rinex/__init__.py | 2 +- tecs/rinex/futils.py | 121 +++-------------------------------------- 6 files changed, 12 insertions(+), 157 deletions(-) diff --git a/README.rst b/README.rst index a99e126..ff5c136 100644 --- a/README.rst +++ b/README.rst @@ -56,20 +56,6 @@ Downloads .. _x86_32: https://github.com/gnss-lab/tec-suite/releases/download/v0.7.8/tec-suite-v0.7.8-linux32.tgz .. _x86_64: https://github.com/gnss-lab/tec-suite/releases/download/v0.7.8/tec-suite-v0.7.8-linux64.tgz -Requirements -~~~~~~~~~~~~ - -``crx2rnx`` - To decompress Hatanaka-compressed RINEX files, **tec-suite** uses - `crx2rnx `_. - -``gunzip`` - To unarchive ``.z``, ``.Z`` or ``.gz``, files **tec-suite** - uses ``gunzip``. If your system is **Linux** or **macOS** you - probably have it installed. You can find the **Windows** version - at `GnuWin `_ - site. - Usage ===== diff --git a/docs/installation.rst b/docs/installation.rst index 639b7aa..90f37d1 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -12,26 +12,3 @@ Downloads: .. _x86_32: https://github.com/gnss-lab/tec-suite/releases/download/v0.7.8/tec-suite-v0.7.8-linux32.tgz .. _x86_64: https://github.com/gnss-lab/tec-suite/releases/download/v0.7.8/tec-suite-v0.7.8-linux64.tgz - - -************ -Requirements -************ - -``crx2rnx`` - To decompress Hatanaka-compressed RINEX files, **tec-suite** uses - `crx2rnx `_. - -``gunzip`` - To unarchive ``.z``, ``.Z`` or ``.gz``, files **tec-sutie** - uses ``gunzip``. If your system is **Linux** or **macOS** you - probably have it installed. You can find the **Windows** version - at `GnuWin `_ - site. - -.. note:: - - **tec-suite** for Windows comes with ``crx2rnx`` and ``gzip`` executables. In - case of Linux or macOS put ``crx2rnx`` to a dir where ``tecs`` could find it, - e.g. to the dir which contains ``tecs`` binary or to any dir in ``$PATH`` - variable. diff --git a/rakefile b/rakefile index ec0297f..9ac1e38 100644 --- a/rakefile +++ b/rakefile @@ -52,12 +52,7 @@ windows? and ( # additional files DIST_EXTRA_FILES_PATH = DISTPATH + '/apps' - DIST_EXTRA_FILES = [ - '_distfiles/apps/crx2rnx.exe', - '_distfiles/apps/gzip.exe', - '_distfiles/apps/crx2rnx.README', - '_distfiles/apps/gzip.README' - ] + DIST_EXTRA_FILES = [] ARCH_CMD = [ '7z', diff --git a/setup.py b/setup.py index 19ba5aa..98f641c 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ py_modules=["tecs"], packages=find_packages(), - install_requires=['future'], + install_requires=['future', 'hatanaka'], extras_require={ 'test': ['coverage', 'nose'], diff --git a/tecs/rinex/__init__.py b/tecs/rinex/__init__.py index 0b555d1..f9812f8 100644 --- a/tecs/rinex/__init__.py +++ b/tecs/rinex/__init__.py @@ -24,7 +24,7 @@ from __future__ import unicode_literals from tecs.rinex.basic import RinexError -from tecs.rinex.futils import RE_VER, GZIP, CRX2RNX, expand_obs, expand_nav +from tecs.rinex.futils import RE_VER, expand_obs, expand_nav from tecs.rinex.header import RinexVersionType from tecs.rinex.v2.o import Obs2, Obs21, Obs211 diff --git a/tecs/rinex/futils.py b/tecs/rinex/futils.py index 2b8ea12..b927af2 100644 --- a/tecs/rinex/futils.py +++ b/tecs/rinex/futils.py @@ -28,7 +28,6 @@ from __future__ import unicode_literals import io -from builtins import map from builtins import str import datetime @@ -37,11 +36,11 @@ import os.path import re -import subprocess import sys -import tempfile from string import Template +import hatanaka + NAME = 'tecs.rinex.futils' RE_VER = re.compile(r'^\s*(\d+\.?\d*).*RINEX VERSION / TYPE$', re.I) @@ -53,7 +52,7 @@ ARCH = r'\.z|\.gz' CRD_EXT = '.xyz' -RE_Z = re.compile(r'(.*)\.(z|gz)$', re.I) +RE_Z = re.compile(r'(.*)\.(z|gz|bz2|zip)$', re.I) RE_CRX = re.compile(r'(.*)\.(\d{2}d|crx)', re.I) RE_RNX = re.compile(r'(.*)\.(\d{2}o|rnx)', re.I) @@ -61,13 +60,6 @@ # RE_CLASSIC_NAV = re.compile(r'^(\w{4})(\d{3})\w\.(\d{2})([nglph])', re.I) # Z_EXT = r'({})$'.format(ARCH) -GZIP = ['gzip', '-cd'] -CRX2RNX = ['crx2rnx', '-'] - -if os.name == 'nt': - GZIP[0] = r'apps/gzip.exe' - CRX2RNX[0] = r'apps/crx2rnx.exe' - def _compose_res(): obs_ext = r'({}|)$'.format(ARCH) @@ -88,8 +80,7 @@ def _compose_res(): re_obs_wo_ext_str = obs_tmpl.substitute(ext='') re_obs_wo_ext_str = '({})'.format(re_obs_wo_ext_str) - return list(map(re.compile, (re_obs_str, re_xyz_str, re_obs_wo_ext_str), - (re.I,) * 3)) + return [re.compile(x, re.I) for x in (re_obs_str, re_xyz_str, re_obs_wo_ext_str)] RE_OBS, RE_XYZ, RE_OBS_WO_EXT = _compose_res() @@ -129,29 +120,8 @@ def expand_nav(filename): ------- f_obj : file """ - - gzip = GZIP + [filename] - - if not RE_Z.match(filename): - return open(filename) - - try: - gzip_pipe = subprocess.Popen( - gzip, - stdout=subprocess.PIPE, - ) - - tmp_file = tempfile.TemporaryFile(mode='w+') - write_ascii_lines(gzip_pipe.stdout, tmp_file) - - gzip_pipe.stdout.close() - - tmp_file.seek(0) - return tmp_file - - except (OSError, ValueError) as err: - msg = "can't uncompress the file (%s)." % err - raise UncompressError(filename, msg) + data = hatanaka.decompress(filename) + return io.StringIO(data.decode('ascii')) def expand_obs(filename): @@ -168,86 +138,13 @@ def expand_obs(filename): f_obj : file """ - gzip = GZIP + [filename] - crx2rnx = CRX2RNX + [filename] - f_bn = os.path.basename(filename) - if not RE_OBS.match(f_bn): + if not RE_OBS.match(f_bn) and not RE_OBS_WO_EXT.match(f_bn): msg = "Not an observation rinex file." raise UncompressError(filename, msg) - del f_bn - - tmp_file = tempfile.TemporaryFile(mode='w+') - - # rinex + .Z - if RE_Z.match(filename): - # gzip - try: - gzip_pipe = subprocess.Popen( - gzip, - stdout=subprocess.PIPE, - ) - except OSError as err: - msg = "can't execute gzip: %s" % str(err) - raise UncompressError(filename, msg) - - # \d{2}o.Z - if RE_RNX.match(filename): - write_ascii_lines(gzip_pipe.stdout, tmp_file) - - # \d{2}d.Z - elif RE_CRX.match(filename): - try: - crx2rnx_pipe = subprocess.Popen( - crx2rnx[0], - stdin=gzip_pipe.stdout, - stdout=subprocess.PIPE, - ) - - except (OSError, ValueError) as err: - msg = "can't execute crx2rnx: %s" % str(err) - raise UncompressError(filename, msg) - - write_ascii_lines(crx2rnx_pipe.stdout, tmp_file) - crx2rnx_pipe.stdout.close() - - gzip_pipe.stdout.close() - - tmp_file.seek(0) - return tmp_file - - # \d{2}d - elif RE_CRX.match(filename): - - try: - crx2rnx_pipe = subprocess.Popen( - crx2rnx, - stdout=subprocess.PIPE, - ) - except (OSError, ValueError) as err: - msg = "can't execute crx2rnx: %s" % str(err) - raise UncompressError(filename, msg) - - write_ascii_lines(crx2rnx_pipe.stdout, tmp_file) - - crx2rnx_pipe.stdout.close() - - tmp_file.seek(0) - return tmp_file - - # \d{2}o - elif RE_RNX.match(filename): - o_file = io.open( - filename, - 'r', - encoding='ascii', - errors='ignore', - ) - return o_file - else: - msg = "Not an observation rinex file." - raise UncompressError(filename, msg) + data = hatanaka.decompress(filename) + return io.StringIO(data.decode('ascii')) def get_rinex_date(filename):