From 6e5dc9225afb98a21ca5f58a666fe96bf84646d0 Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Sat, 17 Mar 2018 17:00:33 +0100 Subject: [PATCH 01/38] Load all files in a data directory: "_all.yml" and other files if present. --- statik/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/statik/database.py b/statik/database.py index 0864955..d201215 100644 --- a/statik/database.py +++ b/statik/database.py @@ -202,8 +202,7 @@ def load_model_data(self, path, model): # try find a model data collection if os.path.isfile(os.path.join(path, '_all.yml')): self.load_model_data_collection(path, model) - else: - self.load_model_data_from_files(path, model) + self.load_model_data_from_files(path, model) self.session.commit() def load_model_data_collection(self, path, model): @@ -264,6 +263,7 @@ def load_model_data_collection(self, path, model): def load_model_data_from_files(self, path, model): db_model = globals()[model.name] entry_files = list_files(path, ['yml', 'yaml', 'md']) + entry_files = [f for f in entry_files if not(f.endswith("_all.yml"))] seen_entries = set() logger.debug("Loading %d instance(s) for model: %s", len(entry_files), model.name) for entry_file in entry_files: From cb2ebf4e26ff336b1ddc59431c4190421ba3990e Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Sun, 18 Mar 2018 09:30:58 +0100 Subject: [PATCH 02/38] Content field from _all.yml files are now loaded. --- statik/common.py | 43 ++++++++++++++++++++++--------------------- statik/database.py | 5 +++-- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/statik/common.py b/statik/common.py index e862887..cc38b08 100644 --- a/statik/common.py +++ b/statik/common.py @@ -56,7 +56,8 @@ class ContentLoadable(object): loading content and metadata from a Markdown file. """ def __init__(self, filename=None, file_type=None, from_string=None, from_dict=None, - name=None, markdown_config=None, encoding='utf-8', error_context=None): + name=None, markdown_config=None, encoding='utf-8', error_context=None, + content_field = None): self.vars = None self.content = None self.file_content = None @@ -102,7 +103,6 @@ def __init__(self, filename=None, file_type=None, from_string=None, from_dict=No "filename", "from_string", "from_dict", context=self.error_context ) - if name is not None: self.name = name elif self.filename is not None: @@ -112,7 +112,23 @@ def __init__(self, filename=None, file_type=None, from_string=None, from_dict=No "name", "filename", context=self.error_context ) - + markdown_ext = [ + MarkdownYamlMetaExtension(), + MarkdownLoremIpsumExtension(error_context=self.error_context) + ] + if self.markdown_config.enable_permalinks: + markdown_ext.append( + MarkdownPermalinkExtension( + permalink_text=self.markdown_config.permalink_text, + permalink_class=self.markdown_config.permalink_class, + permalink_title=self.markdown_config.permalink_title, + ) + ) + markdown_ext.extend(self.markdown_config.extensions) + md = Markdown( + extensions=markdown_ext, + extension_configs=self.markdown_config.extension_config + ) # if it wasn't loaded from a dictionary if self.vars is None: if self.file_type is None: @@ -127,26 +143,11 @@ def __init__(self, filename=None, file_type=None, from_string=None, from_dict=No if not isinstance(self.vars, dict): self.vars = {} else: - markdown_ext = [ - MarkdownYamlMetaExtension(), - MarkdownLoremIpsumExtension(error_context=self.error_context) - ] - if self.markdown_config.enable_permalinks: - markdown_ext.append( - MarkdownPermalinkExtension( - permalink_text=self.markdown_config.permalink_text, - permalink_class=self.markdown_config.permalink_class, - permalink_title=self.markdown_config.permalink_title, - ) - ) - markdown_ext.extend(self.markdown_config.extensions) - - md = Markdown( - extensions=markdown_ext, - extension_configs=self.markdown_config.extension_config - ) self.content = md.convert(self.file_content) self.vars = md.meta + else: + if (content_field is not None) and (content_field in self.vars): + self.content = md.convert(self.vars[content_field]) if isinstance(self.vars, dict): self.vars = dict_strip(self.vars) diff --git a/statik/database.py b/statik/database.py index d201215..b3d2850 100644 --- a/statik/database.py +++ b/statik/database.py @@ -351,7 +351,8 @@ def shutdown(self): class StatikDatabaseInstance(ContentLoadable): def __init__(self, model=None, session=None, **kwargs): - super(StatikDatabaseInstance, self).__init__(**kwargs) + super(StatikDatabaseInstance, self).__init__(content_field = model.content_field, + **kwargs) if model is None: raise MissingParameterError("model", context=self.error_context) self.model = model @@ -362,7 +363,7 @@ def __init__(self, model=None, session=None, **kwargs): # convert the vars to their underscored representation self.field_values = underscore_var_names(self.vars) - self.field_values['pk'] = self.name + self.field_values['pk'] = self.name # run through the foreign key fields to check their assignment for field_name in self.model.field_names: From 44c12a9b1dc7f7152d1055aabbd688b50555e6de Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Sun, 18 Mar 2018 09:30:58 +0100 Subject: [PATCH 03/38] Content field from _all.yml files are now loaded. --- statik/common.py | 43 ++++++++++++++++++++++--------------------- statik/database.py | 5 +++-- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/statik/common.py b/statik/common.py index e862887..cc38b08 100644 --- a/statik/common.py +++ b/statik/common.py @@ -56,7 +56,8 @@ class ContentLoadable(object): loading content and metadata from a Markdown file. """ def __init__(self, filename=None, file_type=None, from_string=None, from_dict=None, - name=None, markdown_config=None, encoding='utf-8', error_context=None): + name=None, markdown_config=None, encoding='utf-8', error_context=None, + content_field = None): self.vars = None self.content = None self.file_content = None @@ -102,7 +103,6 @@ def __init__(self, filename=None, file_type=None, from_string=None, from_dict=No "filename", "from_string", "from_dict", context=self.error_context ) - if name is not None: self.name = name elif self.filename is not None: @@ -112,7 +112,23 @@ def __init__(self, filename=None, file_type=None, from_string=None, from_dict=No "name", "filename", context=self.error_context ) - + markdown_ext = [ + MarkdownYamlMetaExtension(), + MarkdownLoremIpsumExtension(error_context=self.error_context) + ] + if self.markdown_config.enable_permalinks: + markdown_ext.append( + MarkdownPermalinkExtension( + permalink_text=self.markdown_config.permalink_text, + permalink_class=self.markdown_config.permalink_class, + permalink_title=self.markdown_config.permalink_title, + ) + ) + markdown_ext.extend(self.markdown_config.extensions) + md = Markdown( + extensions=markdown_ext, + extension_configs=self.markdown_config.extension_config + ) # if it wasn't loaded from a dictionary if self.vars is None: if self.file_type is None: @@ -127,26 +143,11 @@ def __init__(self, filename=None, file_type=None, from_string=None, from_dict=No if not isinstance(self.vars, dict): self.vars = {} else: - markdown_ext = [ - MarkdownYamlMetaExtension(), - MarkdownLoremIpsumExtension(error_context=self.error_context) - ] - if self.markdown_config.enable_permalinks: - markdown_ext.append( - MarkdownPermalinkExtension( - permalink_text=self.markdown_config.permalink_text, - permalink_class=self.markdown_config.permalink_class, - permalink_title=self.markdown_config.permalink_title, - ) - ) - markdown_ext.extend(self.markdown_config.extensions) - - md = Markdown( - extensions=markdown_ext, - extension_configs=self.markdown_config.extension_config - ) self.content = md.convert(self.file_content) self.vars = md.meta + else: + if (content_field is not None) and (content_field in self.vars): + self.content = md.convert(self.vars[content_field]) if isinstance(self.vars, dict): self.vars = dict_strip(self.vars) diff --git a/statik/database.py b/statik/database.py index d201215..b3d2850 100644 --- a/statik/database.py +++ b/statik/database.py @@ -351,7 +351,8 @@ def shutdown(self): class StatikDatabaseInstance(ContentLoadable): def __init__(self, model=None, session=None, **kwargs): - super(StatikDatabaseInstance, self).__init__(**kwargs) + super(StatikDatabaseInstance, self).__init__(content_field = model.content_field, + **kwargs) if model is None: raise MissingParameterError("model", context=self.error_context) self.model = model @@ -362,7 +363,7 @@ def __init__(self, model=None, session=None, **kwargs): # convert the vars to their underscored representation self.field_values = underscore_var_names(self.vars) - self.field_values['pk'] = self.name + self.field_values['pk'] = self.name # run through the foreign key fields to check their assignment for field_name in self.model.field_names: From 725a57712b9a21e5b6e397558f7071b8979957d4 Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Mon, 19 Mar 2018 15:46:41 +0100 Subject: [PATCH 04/38] Check if field is present in case of ManyToMany relation --- statik/database.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/statik/database.py b/statik/database.py index b3d2850..9e996e2 100644 --- a/statik/database.py +++ b/statik/database.py @@ -379,7 +379,7 @@ def __init__(self, model=None, session=None, **kwargs): self.field_values['%s_id' % field_name] = self.field_values[field_name] del self.field_values[field_name] - elif isinstance(field, StatikManyToManyField): + elif isinstance(field, StatikManyToManyField) and (field_name in self.field_values): if not isinstance(self.field_values[field_name], list): raise InvalidFieldTypeError( self.model.name, @@ -387,7 +387,6 @@ def __init__(self, model=None, session=None, **kwargs): "a list", context=self.error_context ) - logger.debug( "Attempting to look up primary keys for ManyToMany " + "field relationship: %s", self.field_values[field_name] From 5076a9bf462114aae8debffd2e1efabd7a227791 Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Sat, 2 Mar 2019 19:45:45 +0100 Subject: [PATCH 05/38] clean --- examples/blog/models/Tag.yml | 0 tests/integration/data-non-root-base/assets/EMPTY | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 examples/blog/models/Tag.yml delete mode 100644 tests/integration/data-non-root-base/assets/EMPTY diff --git a/examples/blog/models/Tag.yml b/examples/blog/models/Tag.yml deleted file mode 100644 index e69de29..0000000 diff --git a/tests/integration/data-non-root-base/assets/EMPTY b/tests/integration/data-non-root-base/assets/EMPTY deleted file mode 100644 index e69de29..0000000 From 4a96b8e4dc6b367a87b97f5a7a82e9d428b637d5 Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Sat, 2 Mar 2019 19:47:09 +0100 Subject: [PATCH 06/38] Change yaml load to full_load. See https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation --- statik/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statik/database.py b/statik/database.py index 9e996e2..c630d10 100644 --- a/statik/database.py +++ b/statik/database.py @@ -212,7 +212,7 @@ def load_model_data_collection(self, path, model): db_model = globals()[model.name] # load the collection data from the collection file with open(full_filename, mode='rt', encoding=self.encoding) as f: - collection = yaml.load(f.read()) + collection = yaml.full_load(f.read()) if not isinstance(collection, list): raise InvalidModelCollectionDataError( From cc5259b97a97a2c2f07f7f3e56d736c392ff8349 Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Sun, 3 Mar 2019 19:25:03 +0100 Subject: [PATCH 07/38] Ignore pipfile --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 035941a..231f22c 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,7 @@ pip-selfcheck.json # Generated folders public/ + +Pipfile + +Pipfile\.lock From cd71362f771ad27dda832cb69adabeae28068f9e Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Sun, 3 Mar 2019 19:25:56 +0100 Subject: [PATCH 08/38] Disable markdown extension --- statik/markdown_config.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/statik/markdown_config.py b/statik/markdown_config.py index 96c385d..8ba07fe 100644 --- a/statik/markdown_config.py +++ b/statik/markdown_config.py @@ -14,10 +14,11 @@ class MarkdownConfig(object): DEFAULT_MARKDOWN_EXTENSIONS = [ - 'markdown.extensions.fenced_code', - 'markdown.extensions.tables', - 'markdown.extensions.toc', - 'markdown.extensions.footnotes' + # 'markdown.extensions.fenced_code', + # 'markdown.extensions.tables', + # 'markdown.extensions.toc', + # 'markdown.extensions.footnotes' + # NoteExtension() ] def __init__(self, markdown_params=None, error_context=None): From 326c0e39eabf8c1cb90965782d236a36125648b7 Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Sat, 2 Mar 2019 19:47:09 +0100 Subject: [PATCH 09/38] Revert "Change yaml load to full_load. See https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation" This reverts commit 4a96b8e4dc6b367a87b97f5a7a82e9d428b637d5. --- statik/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statik/database.py b/statik/database.py index c630d10..9e996e2 100644 --- a/statik/database.py +++ b/statik/database.py @@ -212,7 +212,7 @@ def load_model_data_collection(self, path, model): db_model = globals()[model.name] # load the collection data from the collection file with open(full_filename, mode='rt', encoding=self.encoding) as f: - collection = yaml.full_load(f.read()) + collection = yaml.load(f.read()) if not isinstance(collection, list): raise InvalidModelCollectionDataError( From bcec899a76a2da7e9753f22910e78d8c39601c5b Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Sat, 2 Mar 2019 19:47:09 +0100 Subject: [PATCH 10/38] Change yaml load to full_load. See https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation --- statik/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statik/database.py b/statik/database.py index 9e996e2..c630d10 100644 --- a/statik/database.py +++ b/statik/database.py @@ -212,7 +212,7 @@ def load_model_data_collection(self, path, model): db_model = globals()[model.name] # load the collection data from the collection file with open(full_filename, mode='rt', encoding=self.encoding) as f: - collection = yaml.load(f.read()) + collection = yaml.full_load(f.read()) if not isinstance(collection, list): raise InvalidModelCollectionDataError( From eb576d1645704ad20f4c06052b81cdbf9b71a354 Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Mon, 4 Mar 2019 16:41:16 +0100 Subject: [PATCH 11/38] =?UTF-8?q?Logging=20des=20donn=C3=A9es=20non=20trou?= =?UTF-8?q?v=C3=A9es=20lors=20du=20build?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- statik/database.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/statik/database.py b/statik/database.py index c630d10..28275da 100644 --- a/statik/database.py +++ b/statik/database.py @@ -11,6 +11,7 @@ from sqlalchemy import String, Integer, Column, Table, ForeignKey, \ Boolean, DateTime, Text, create_engine from sqlalchemy.orm import sessionmaker, relationship, backref +from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.ext.declarative import declarative_base import mlalchemy @@ -357,6 +358,8 @@ def __init__(self, model=None, session=None, **kwargs): raise MissingParameterError("model", context=self.error_context) self.model = model + self.implicit_data_items = [] + if session is None: raise MissingParameterError("session", context=self.error_context) self.session = session @@ -394,12 +397,43 @@ def __init__(self, model=None, session=None, **kwargs): # convert the list of field values to a query to look up the # primary keys of the corresponding table other_model = globals()[field.field_type] + + missing_items = [] + + for item in self.field_values[field_name]: + try: + self.session.query( + other_model + ).filter( + other_model.pk == item + ).one() + except NoResultFound: + # Only allow implicit tables if the model + # has no fields other than 'pk' + if len(other_model.__table__._columns) == 1: + missing_items.append({'pk': item}) + else: + logger.warning('%s not found in %s' % + (item, other_model)) + else: + logger.debug('%s found', item) + + self.implicit_data_items.append( + (field.field_type, missing_items) + ) + + original_values = self.field_values[field_name] + self.field_values[field_name] = self.session.query( other_model ).filter( other_model.pk.in_(self.field_values[field_name]) ).all() + # Ensure that values appear in original order + self.field_values[field_name].sort( + key=lambda x: original_values.index(x.pk)) + # populate any Content field for this model if self.model.content_field is not None: self.field_values[self.model.content_field] = self.content From 31b882b56f85575290af65c5299a717c1500e3ea Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Fri, 8 Mar 2019 15:47:21 +0100 Subject: [PATCH 12/38] clean --- files.txt | 10 ++++++++++ statik/database.py | 2 +- statik/markdown_config.py | 35 +++++++++++++++++++---------------- 3 files changed, 30 insertions(+), 17 deletions(-) create mode 100644 files.txt diff --git a/files.txt b/files.txt new file mode 100644 index 0000000..db603ce --- /dev/null +++ b/files.txt @@ -0,0 +1,10 @@ +/usr/local/lib/python3.7/site-packages/statik-0.21.2-py3.7.egg +/usr/local/bin/statik +/usr/local/bin/pystache +/usr/local/bin/pystache-test +/usr/local/bin/httpwatcher +/usr/local/bin/slugify +/usr/local/bin/markdown_py +/usr/local/bin/futurize +/usr/local/bin/pasteurize +/usr/local/bin/watchmedo diff --git a/statik/database.py b/statik/database.py index 28275da..0c53b94 100644 --- a/statik/database.py +++ b/statik/database.py @@ -213,7 +213,7 @@ def load_model_data_collection(self, path, model): db_model = globals()[model.name] # load the collection data from the collection file with open(full_filename, mode='rt', encoding=self.encoding) as f: - collection = yaml.full_load(f.read()) + collection = yaml.load(f.read()) if not isinstance(collection, list): raise InvalidModelCollectionDataError( diff --git a/statik/markdown_config.py b/statik/markdown_config.py index 8ba07fe..5faf7f0 100644 --- a/statik/markdown_config.py +++ b/statik/markdown_config.py @@ -6,9 +6,7 @@ from statik.errors import * -__all__ = [ - 'MarkdownConfig' -] +__all__ = ["MarkdownConfig"] class MarkdownConfig(object): @@ -27,57 +25,62 @@ def __init__(self, markdown_params=None, error_context=None): if not isinstance(markdown_params, dict): raise ProjectConfigurationError( message="Markdown configuration parameters must be a dictionary.", - context=error_context + context=error_context, ) - permalinks_config = markdown_params.get('permalinks', dict()) + permalinks_config = markdown_params.get("permalinks", dict()) - self.enable_permalinks = permalinks_config.get('enabled', False) + self.enable_permalinks = permalinks_config.get("enabled", False) if self.enable_permalinks in {"true", "1", 1}: self.enable_permalinks = True elif self.enable_permalinks in {"false", "0", 0}: self.enable_permalinks = False - self.permalink_text = permalinks_config.get('text', "¶") - self.permalink_class = permalinks_config.get('class', None) - self.permalink_title = permalinks_config.get('title', None) + self.permalink_text = permalinks_config.get("text", "¶") + self.permalink_class = permalinks_config.get("class", None) + self.permalink_title = permalinks_config.get("title", None) # Required list of Markdown extensions self.extensions = copy(MarkdownConfig.DEFAULT_MARKDOWN_EXTENSIONS) # Configuration for the markdown extensions self.extension_config = {} - extension_list = markdown_params.get('extensions', []) + extension_list = markdown_params.get("extensions", []) # if it's a dictionary, first convert it to our list notation if isinstance(extension_list, dict): extension_list = [] - for ext_package, config in iteritems(markdown_params['extensions']): + for ext_package, config in iteritems(markdown_params["extensions"]): extension_list.append({ext_package: config}) # Try to load extensions as requested by config for extension in extension_list: if isinstance(extension, dict): - ext_package, config = next(iter(extension.keys())), next(iter(extension.values())) + ext_package, config = ( + next(iter(extension.keys())), + next(iter(extension.values())), + ) else: ext_package, config = extension, None if ext_package not in self.extensions: self.extensions.append(ext_package) - if config is not None: + if config is not None: if ext_package in self.extension_config: self.extension_config[ext_package].update(config) else: self.extension_config[ext_package] = config def __repr__(self): - return ("MarkdownConfig(enable_permalinks=%s, permalink_text=%s, permalink_class=%s, " + - "permalink_title=%s, extensions=%s, extension_config=%s)") % ( + return ( + "MarkdownConfig(enable_permalinks=%s, permalink_text=%s, permalink_class=%s, " + + "permalink_title=%s, extensions=%s, extension_config=%s)" + ) % ( self.enable_permalinks, self.permalink_text, self.permalink_class, self.permalink_title, self.extensions, - self.extension_config + self.extension_config, ) From 7299f6b7e0db392515aaa2289a3ab1aeb2b53124 Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Fri, 8 Mar 2019 17:42:53 +0100 Subject: [PATCH 13/38] Add warnings --- statik/database.py | 20 +++++++++++++++++++- statik/utils.py | 26 +++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/statik/database.py b/statik/database.py index 0c53b94..71cb9fe 100644 --- a/statik/database.py +++ b/statik/database.py @@ -213,7 +213,7 @@ def load_model_data_collection(self, path, model): db_model = globals()[model.name] # load the collection data from the collection file with open(full_filename, mode='rt', encoding=self.encoding) as f: - collection = yaml.load(f.read()) + collection = yaml.load(f.read(), Loader=yaml.FullLoader) if not isinstance(collection, list): raise InvalidModelCollectionDataError( @@ -394,6 +394,24 @@ def __init__(self, model=None, session=None, **kwargs): "Attempting to look up primary keys for ManyToMany " + "field relationship: %s", self.field_values[field_name] ) + + duplicates_in_array = find_duplicates_in_array(self.field_values[field_name]) + + if duplicates_in_array: + logger.warning("Duplicates found in %s: %s (field: %s)", + self.filename, + duplicates_in_array, + field_name) + self.field_values[field_name] = list(set(self.field_values[field_name])) + + # check if non-string items are present + for item in self.field_values[field_name]: + if not isinstance(item, ("".__class__, u"".__class__)): + logger.warning("Non-string values found in array " + + "(field: %s, instance: %s, model: %s): %s", + field_name, self.field_values['pk'], self.model.name, item) + + # convert the list of field values to a query to look up the # primary keys of the corresponding table other_model = globals()[field.field_type] diff --git a/statik/utils.py b/statik/utils.py index 5d83d65..2c4f7d0 100644 --- a/statik/utils.py +++ b/statik/utils.py @@ -42,7 +42,8 @@ '_str', '_unicode', 'find_first_file_with_ext', - 'uncapitalize' + 'uncapitalize', + 'find_duplicates_in_array', ] DEFAULT_CONFIG_CONTENT = """project-name: Your project name @@ -357,3 +358,26 @@ def find_first_file_with_ext(base_paths, prefix, exts): def uncapitalize(s): """If the given string begins with a capital letter, it converts it to lowercase.""" return (s[:1].lower() + s[1:]) if s else "" + +def find_duplicates_in_array(array): + """Runs through the array and returns the elements that contain + more than one duplicate + Args: + array: The array to check for duplicates. + Returns: + Array of the elements that are duplicates. Returns empty list if + there are no duplicates. + """ + duplicates = [] + non_duplicates = [] + + if len(array) != len(set(array)): + for item in array: + if item not in non_duplicates: + non_duplicates.append(item) + elif item in non_duplicates and item not in duplicates: + duplicates.append(item) + + return duplicates + + \ No newline at end of file From 432452644d029767f38f7129c93a850f64c0136e Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Sat, 12 Oct 2019 18:03:26 +0200 Subject: [PATCH 14/38] Changement dans la gestion de _all.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Désormais les fichiers *_.yml sont chargés comme les _all.yml. Il peut donc y en avoir plusieurs. --- statik/database.py | 94 +++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 46 deletions(-) diff --git a/statik/database.py b/statik/database.py index 71cb9fe..eeee432 100644 --- a/statik/database.py +++ b/statik/database.py @@ -6,6 +6,7 @@ from io import open import os.path +import glob import yaml from sqlalchemy import String, Integer, Column, Table, ForeignKey, \ @@ -201,70 +202,71 @@ def load_model_data(self, path, model): """ if os.path.isdir(path): # try find a model data collection - if os.path.isfile(os.path.join(path, '_all.yml')): + if glob.glob(os.path.join(path, "*_.yml")): self.load_model_data_collection(path, model) self.load_model_data_from_files(path, model) self.session.commit() def load_model_data_collection(self, path, model): - full_filename = os.path.join(path, '_all.yml') - self.error_context.update(filename=full_filename) + for full_filename in glob.iglob(os.path.join(path, "*_.yml")): + # full_filename = os.path.join(path, '_all.yml') + self.error_context.update(filename=full_filename) - db_model = globals()[model.name] - # load the collection data from the collection file - with open(full_filename, mode='rt', encoding=self.encoding) as f: - collection = yaml.load(f.read(), Loader=yaml.FullLoader) + db_model = globals()[model.name] + # load the collection data from the collection file + with open(full_filename, mode='rt', encoding=self.encoding) as f: + collection = yaml.load(f.read(), Loader=yaml.FullLoader) - if not isinstance(collection, list): - raise InvalidModelCollectionDataError( - model.name, - context=self.error_context - ) - seen_entries = set() - logger.debug("Loading %d instance(s) for model: %s", len(collection), model.name) - for item in collection: - if not isinstance(item, dict) or 'pk' not in item: + if not isinstance(collection, list): raise InvalidModelCollectionDataError( model.name, context=self.error_context ) + seen_entries = set() + logger.debug("Loading %d instance(s) for model: %s", len(collection), model.name) + for item in collection: + if not isinstance(item, dict) or 'pk' not in item: + raise InvalidModelCollectionDataError( + model.name, + context=self.error_context + ) - entry = StatikDatabaseInstance( - name=item['pk'], - from_dict=item, - model=model, - session=self.session, - encoding=self.encoding, - markdown_config=self.markdown_config - ) - # duplicate primary key! - if entry.field_values['pk'] in seen_entries: - raise DuplicateModelInstanceError( - model.name, - pk=entry.field_values['pk'], - context=self.error_context - ) - else: - seen_entries.add(entry.field_values['pk']) - - try: - db_entry = db_model(**entry.field_values) - self.session.add(db_entry) - except Exception as exc: - raise DataError( - model.name, - pk=entry.field_values['pk'], - message="failed to insert entry into in-memory database.", - orig_exc=exc, - context=self.error_context + entry = StatikDatabaseInstance( + name=item['pk'], + from_dict=item, + model=model, + session=self.session, + encoding=self.encoding, + markdown_config=self.markdown_config ) - + # duplicate primary key! + if entry.field_values['pk'] in seen_entries: + raise DuplicateModelInstanceError( + model.name, + pk=entry.field_values['pk'], + context=self.error_context + ) + else: + seen_entries.add(entry.field_values['pk']) + + try: + db_entry = db_model(**entry.field_values) + self.session.add(db_entry) + except Exception as exc: + raise DataError( + model.name, + pk=entry.field_values['pk'], + message="failed to insert entry into in-memory database.", + orig_exc=exc, + context=self.error_context + ) + self.error_context.clear() def load_model_data_from_files(self, path, model): db_model = globals()[model.name] entry_files = list_files(path, ['yml', 'yaml', 'md']) - entry_files = [f for f in entry_files if not(f.endswith("_all.yml"))] + entry_files = [f for f in entry_files if not(f.endswith("_.yml"))] seen_entries = set() logger.debug("Loading %d instance(s) for model: %s", len(entry_files), model.name) for entry_file in entry_files: From 9db3ae2582863f1c9d5e843a9947dac704d4e7b3 Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Wed, 22 Jan 2020 18:25:52 +0100 Subject: [PATCH 15/38] =?UTF-8?q?La=20version=20pr=C3=A9cise=20qu'on=20est?= =?UTF-8?q?=20en=20"PP"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 ++ statik/__init__.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 231f22c..1aaaed4 100644 --- a/.gitignore +++ b/.gitignore @@ -106,3 +106,5 @@ public/ Pipfile Pipfile\.lock + +.DS_Store \ No newline at end of file diff --git a/statik/__init__.py b/statik/__init__.py index 95d4256..3f39ecf 100644 --- a/statik/__init__.py +++ b/statik/__init__.py @@ -1,3 +1,3 @@ # -*- coding:utf-8 -*- -__version__ = "0.21.2" +__version__ = "0.21.2p" From 4541b11d953637a6e9840437c9d3009be75c6571 Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Mon, 10 Feb 2020 09:12:10 +0100 Subject: [PATCH 16/38] Change in name of *.yml --- .gitignore | 3 ++- statik/database.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 1aaaed4..8ef5b05 100644 --- a/.gitignore +++ b/.gitignore @@ -107,4 +107,5 @@ Pipfile Pipfile\.lock -.DS_Store \ No newline at end of file +.DS_Store +statik.code-workspace diff --git a/statik/database.py b/statik/database.py index eeee432..421429d 100644 --- a/statik/database.py +++ b/statik/database.py @@ -202,13 +202,13 @@ def load_model_data(self, path, model): """ if os.path.isdir(path): # try find a model data collection - if glob.glob(os.path.join(path, "*_.yml")): + if glob.glob(os.path.join(path, "*.yml")): self.load_model_data_collection(path, model) self.load_model_data_from_files(path, model) self.session.commit() def load_model_data_collection(self, path, model): - for full_filename in glob.iglob(os.path.join(path, "*_.yml")): + for full_filename in glob.iglob(os.path.join(path, "*.yml")): # full_filename = os.path.join(path, '_all.yml') self.error_context.update(filename=full_filename) @@ -266,7 +266,7 @@ def load_model_data_collection(self, path, model): def load_model_data_from_files(self, path, model): db_model = globals()[model.name] entry_files = list_files(path, ['yml', 'yaml', 'md']) - entry_files = [f for f in entry_files if not(f.endswith("_.yml"))] + entry_files = [f for f in entry_files if not(f.endswith(".yml"))] seen_entries = set() logger.debug("Loading %d instance(s) for model: %s", len(entry_files), model.name) for entry_file in entry_files: From 2f3ac00e4d4fdfcbe90a228bda53a259323a74fa Mon Sep 17 00:00:00 2001 From: Patrick Paul Date: Wed, 14 Feb 2018 13:43:27 -0500 Subject: [PATCH 17/38] Fixes #64 - last tag registered was invoked for all tag invocations --- statik/jinja2ext.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/statik/jinja2ext.py b/statik/jinja2ext.py index ec53d3e..91d199c 100644 --- a/statik/jinja2ext.py +++ b/statik/jinja2ext.py @@ -146,8 +146,8 @@ def __init__(self, environment): self.active_tag = None logger.debug("Loaded custom template tags: %s", ", ".join(self.tags)) - def _invoke_tag(self, context, *args, **kwargs): - return templatetags.store.invoke_tag(self.active_tag, context, *args, **kwargs) + def _invoke_tag(self, tag_name, context, *args, **kwargs): + return templatetags.store.invoke_tag(tag_name, context, *args, **kwargs) def parse(self, parser): lineno = next(parser.stream).lineno @@ -165,6 +165,7 @@ def parse(self, parser): # get the tag_name for use in looking up callable self.active_tag = parser._tag_stack[-1] + args.insert(0, nodes.Const(self.active_tag)) # create the node node = self.call_method('_invoke_tag', args=args, lineno=lineno) From 6932b5eeadfe97e9524aa1b51988f87d046d9cfb Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Thu, 15 Feb 2018 09:29:15 +0200 Subject: [PATCH 18/38] bumping version to v0.21.3 --- statik/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/statik/__init__.py b/statik/__init__.py index 3f39ecf..b27a698 100644 --- a/statik/__init__.py +++ b/statik/__init__.py @@ -1,3 +1,4 @@ # -*- coding:utf-8 -*- -__version__ = "0.21.2p" +__version__ = "0.21.3" + From 5758022865d2dc469aad32a190350c4293736c0c Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Thu, 15 Feb 2018 09:31:08 +0200 Subject: [PATCH 19/38] adding CHANGELOG entry for v0.21.3 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1444bb2..eab70f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ This is the **Statik** change log as of version `0.6.0`. ## Release History +### `v0.21.3` - 15 February 2018 + +* Merging #65 to fix #64. + ### `v0.21.2` - 12 February 2018 * Attempting to fix issues #50 and #63 - automatic translation of special Unicode characters From 42f455ad54b53cc8f9e89afc3e7cd8cc6b4a25f9 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Mon, 8 Oct 2018 16:06:00 +0200 Subject: [PATCH 20/38] Updates code to allow for newest version of Markdown library (> 3.0) --- statik/markdown_exts.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/statik/markdown_exts.py b/statik/markdown_exts.py index 85290e6..20f84f4 100644 --- a/statik/markdown_exts.py +++ b/statik/markdown_exts.py @@ -28,11 +28,11 @@ class MarkdownYamlMetaExtension(Extension): - def extendMarkdown(self, md, md_globals): - md.preprocessors.add( - 'yaml-meta', + def extendMarkdown(self, md): + md.preprocessors.register( MarkdownYamlMetaPreprocessor(md), - ">normalize_whitespace", + 'yaml-meta', + 40 ) @@ -44,16 +44,16 @@ def __init__(self, *args, **kwargs): self.permalink_title = kwargs.pop('permalink_title', None) super(MarkdownPermalinkExtension, self).__init__(*args, **kwargs) - def extendMarkdown(self, md, md_globals): - md.treeprocessors.add( - 'permalink', + def extendMarkdown(self, md): + md.treeprocessors.register( MarkdownPermalinkProcessor( md, permalink_text=self.permalink_text, permalink_class=self.permalink_class, permalink_title=self.permalink_title ), - 'yaml-meta" + "lipsum", + 50 ) @@ -75,7 +75,7 @@ class MarkdownYamlMetaPreprocessor(Preprocessor): def run(self, lines): result = [] - self.markdown.meta = {} + self.md.meta = {} if len(lines) > 1: yaml_lines = [] @@ -96,7 +96,7 @@ def run(self, lines): result.append(line) if len(yaml_lines) > 0: - self.markdown.meta = yaml.safe_load( + self.md.meta = yaml.safe_load( '\n'.join(yaml_lines) ) From 91bd57f20e65e724e8e228157abc6b0e3c3c87a5 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Mon, 8 Oct 2018 16:06:13 +0200 Subject: [PATCH 21/38] Fixes requirements to latest versions of libraries --- requirements.txt | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8b7889d..5b812c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,20 @@ -future>=0.16.0 -jinja2>=2.8 -PyYAML>=3.11 -SQLAlchemy>=1.0.14 -markdown>=2.6.6 -python-slugify>=1.2.1 -six>=1.10.0 -lipsum>=0.1.1 -httpwatcher>=0.5.1 -mlalchemy>=0.2.1 -pystache>=0.5.4 -colorlog>=3.1.0 -python-dateutil>=2.6.1 \ No newline at end of file +argh==0.26.2 +colorlog==3.1.4 +future==0.16.0 +httpwatcher==0.5.1 +Jinja2==2.10 +lipsum==0.1.2 +Markdown==3.0.1 +MarkupSafe==1.0 +mlalchemy==0.2.2 +pathtools==0.1.2 +pkg-resources==0.0.0 +pystache==0.5.4 +python-dateutil==2.7.3 +python-slugify==1.2.6 +PyYAML==3.13 +six==1.11.0 +SQLAlchemy==1.2.12 +tornado==5.1.1 +Unidecode==1.0.22 +watchdog==0.9.0 From 87aae7c7389e1cbb126a64797a5466ffbf1cca45 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Mon, 8 Oct 2018 16:09:28 +0200 Subject: [PATCH 22/38] Removes broken import --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5b812c3..2354fd6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,6 @@ Markdown==3.0.1 MarkupSafe==1.0 mlalchemy==0.2.2 pathtools==0.1.2 -pkg-resources==0.0.0 pystache==0.5.4 python-dateutil==2.7.3 python-slugify==1.2.6 From 290adb3e43f55ebcedb96948a739bf0999ac499a Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Mon, 8 Oct 2018 16:16:17 +0200 Subject: [PATCH 23/38] Updates changelog for v0.22.0 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eab70f8..a464102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ This is the **Statik** change log as of version `0.6.0`. ## Release History +### `v0.22.0` - 8 October 2018 + +* Merges #68 +* Merges #73 to add SFTP upload functionality +* Updates dependency versions in `requirements.txt` +* Updates Markdown package integration to use new API + ### `v0.21.3` - 15 February 2018 * Merging #65 to fix #64. From 6c2484346dcca1f1dce36f43e735d26789d94f20 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Mon, 8 Oct 2018 16:18:14 +0200 Subject: [PATCH 24/38] Updates version to v0.22.0 --- statik/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statik/__init__.py b/statik/__init__.py index b27a698..b9dca65 100644 --- a/statik/__init__.py +++ b/statik/__init__.py @@ -1,4 +1,4 @@ # -*- coding:utf-8 -*- -__version__ = "0.21.3" +__version__ = "0.22.0" From 204b3fdf96e3733f3d6be467ffbaedb349c59a4e Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Mon, 8 Oct 2018 16:19:54 +0200 Subject: [PATCH 25/38] Fixes missing paramiko requirement --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 2354fd6..ef30d27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +wheel argh==0.26.2 colorlog==3.1.4 future==0.16.0 @@ -17,3 +18,4 @@ SQLAlchemy==1.2.12 tornado==5.1.1 Unidecode==1.0.22 watchdog==0.9.0 +paramiko==2.4.2 From 9cc0fa3bc205b0bd139898ecf32071ff621354a1 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Mon, 8 Oct 2018 16:20:16 +0200 Subject: [PATCH 26/38] Bumps version to v0.22.1 --- statik/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statik/__init__.py b/statik/__init__.py index b9dca65..14c1fb5 100644 --- a/statik/__init__.py +++ b/statik/__init__.py @@ -1,4 +1,4 @@ # -*- coding:utf-8 -*- -__version__ = "0.22.0" +__version__ = "0.22.1" From 84c44c141ffa77b2ae05afa972b246cc336de5b3 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Mon, 8 Oct 2018 16:21:06 +0200 Subject: [PATCH 27/38] Updates changelog for v0.22.1 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a464102..f9f8ae9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ This is the **Statik** change log as of version `0.6.0`. ## Release History +### `v0.22.1` - 8 October 2018 + +* Minor fix to add missing `paramiko` dependency + ### `v0.22.0` - 8 October 2018 * Merges #68 From ee76604f1b4d3de815f5b5ecff5d1bca12bb150a Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Sat, 20 Oct 2018 06:44:32 +0200 Subject: [PATCH 28/38] Updates CHANGELOG for v0.22.2 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9f8ae9..deed71d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ This is the **Statik** change log as of version `0.6.0`. ## Release History +### `v0.22.2` - 20 October 2018 + +* Merging #74 to better organise the growing list of CLI arguments. + ### `v0.22.1` - 8 October 2018 * Minor fix to add missing `paramiko` dependency From d139b0e4ee0e98970d30cba4bbe96fc19bf22788 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Sat, 20 Oct 2018 06:44:47 +0200 Subject: [PATCH 29/38] Bumps version to v0.22.2 --- statik/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statik/__init__.py b/statik/__init__.py index 14c1fb5..14a6482 100644 --- a/statik/__init__.py +++ b/statik/__init__.py @@ -1,4 +1,4 @@ # -*- coding:utf-8 -*- -__version__ = "0.22.1" +__version__ = "0.22.2" From 625b9beec4b8e2ddcb7cedc31f5d620cf2148a60 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Sat, 20 Oct 2018 06:47:31 +0200 Subject: [PATCH 30/38] Adds Python 3.7 to Travis CI build --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c85ed80..14889e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ python: - "2.7" - "3.5" - "3.6" + - "3.7" install: - "pip install -r requirements.txt" script: From 333037f756901eee6af7d244d82c51dd52a7f0ef Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Sat, 20 Oct 2018 06:54:49 +0200 Subject: [PATCH 31/38] Tries Tornado Travis CI approach to be able to build Python 3.7 --- .travis.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 14889e7..2815769 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,22 @@ language: python -python: - - "2.7" - - "3.5" - - "3.6" - - "3.7" + +# Takes inspiration from https://github.com/tornadoweb/tornado/blob/master/.travis.yml +.mixins: +- &xenial-mixin + dist: xenial + sudo: true + addons: + apt: + packages: + - libgnutls-dev +jobs: + include: + - python: 2.7 + - python: 3.5 + - python: 3.6 + - <<: *xenial-mixin + python: 3.7 + install: - "pip install -r requirements.txt" script: From c6eb4cb09a503d5e1ade7f66fe86f67e0dedfb63 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Sat, 20 Oct 2018 07:06:27 +0200 Subject: [PATCH 32/38] Removes CONTRIBUTORS file in favour of just going to https://github.com/thanethomson/statik/graphs/contributors (less overhead) --- CONTRIBUTORS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index c7126f1..003f368 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -1,4 +1,5 @@ Thane Thomson Patrick Paul Kenton Hamaluik -Roman Vaughan \ No newline at end of file +Roman Vaughan +konstruktiv From c530c6958f5b291a0738c138e0591bb1ffcaa0eb Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Sat, 20 Oct 2018 07:06:45 +0200 Subject: [PATCH 33/38] Removes CONTRIBUTORS file in favour of just going to https://github.com/thanethomson/statik/graphs/contributors (less overhead) --- CONTRIBUTORS | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 CONTRIBUTORS diff --git a/CONTRIBUTORS b/CONTRIBUTORS deleted file mode 100644 index 003f368..0000000 --- a/CONTRIBUTORS +++ /dev/null @@ -1,5 +0,0 @@ -Thane Thomson -Patrick Paul -Kenton Hamaluik -Roman Vaughan -konstruktiv From f6810f3de77108601cf2f7cbd83f2cb06c636cc0 Mon Sep 17 00:00:00 2001 From: Thane Thomson Date: Sat, 20 Oct 2018 07:08:10 +0200 Subject: [PATCH 34/38] Updates package tags to include Python 3.7 for next release --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 3e59b8e..7831e82 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ def get_version(): "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Topic :: Internet :: WWW/HTTP", "Topic :: Utilities", ] From 4d37d1b9ed860dedfa5cbff723cf6b5303a1e5b3 Mon Sep 17 00:00:00 2001 From: Paul Pichaureau Date: Sat, 17 Mar 2018 17:00:33 +0100 Subject: [PATCH 35/38] Load all files in a data directory: "_all.yml" and other files if present. --- statik/database.py | 340 ++++++++++++++++++++++++--------------------- 1 file changed, 184 insertions(+), 156 deletions(-) diff --git a/statik/database.py b/statik/database.py index 421429d..aabf606 100644 --- a/statik/database.py +++ b/statik/database.py @@ -9,8 +9,17 @@ import glob import yaml -from sqlalchemy import String, Integer, Column, Table, ForeignKey, \ - Boolean, DateTime, Text, create_engine +from sqlalchemy import ( + String, + Integer, + Column, + Table, + ForeignKey, + Boolean, + DateTime, + Text, + create_engine, +) from sqlalchemy.orm import sessionmaker, relationship, backref from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.ext.declarative import declarative_base @@ -33,40 +42,43 @@ import math import logging + logger = logging.getLogger(__name__) __all__ = [ - 'StatikDatabase', + "StatikDatabase", ] SQLALCHEMY_FIELD_MAPPER = { - 'String': String, - 'DateTime': DateTime, - 'Integer': Integer, - 'Boolean': Boolean, - 'Content': Text, - 'Text': Text + "String": String, + "DateTime": DateTime, + "Integer": Integer, + "Boolean": Boolean, + "Content": Text, + "Text": Text, } def set_global(name, val): - logger.debug('Setting global: %s = %s' % (name, val)) + logger.debug("Setting global: %s = %s" % (name, val)) globals()[name] = val set_global.tracked_globals.add(name) + + set_global.tracked_globals = set() def clear_tracked_globals(): for name in set_global.tracked_globals: - logger.debug('Clearing tracked global: %s' % name) + logger.debug("Clearing tracked global: %s" % name) del globals()[name] set_global.tracked_globals = set() class StatikDatabase(object): - - def __init__(self, data_path, models, encoding=None, markdown_config=None, - error_context=None): + def __init__( + self, data_path, models, encoding=None, markdown_config=None, error_context=None + ): """Constructor. Args: @@ -83,16 +95,16 @@ def __init__(self, data_path, models, encoding=None, markdown_config=None, self.models = models self.markdown_config = markdown_config self.error_context = error_context or StatikErrorContext() - self.engine = create_engine('sqlite:///:memory:') + self.engine = create_engine("sqlite:///:memory:") self.Base = declarative_base() self.session = sessionmaker(bind=self.engine)() - set_global('session', self.session) + set_global("session", self.session) self.find_backrefs() self.create_db(models) def find_backrefs(self): for model_name, model in iteritems(self.models): - logger.debug('Attempting to find backrefs for model: %s', model_name) + logger.debug("Attempting to find backrefs for model: %s", model_name) try: model.find_additional_rels(self.models) except Exception as exc: @@ -100,7 +112,7 @@ def find_backrefs(self): model_name, message="failed to accurately determine model cross-referencing.", orig_exc=exc, - context=self.error_context + context=self.error_context, ) def create_db(self, models): @@ -119,8 +131,7 @@ def create_db(self, models): self.Base.metadata.create_all(self.engine) except Exception as exc: raise StatikError( - message="Failed to create in-memory data model.", - orig_exc=exc + message="Failed to create in-memory data model.", orig_exc=exc ) self.load_all_model_data(models) @@ -146,7 +157,9 @@ def sort_models(self): A sorted list containing the names of the models. """ model_names = [ - table.name for table in self.Base.metadata.sorted_tables if table.name in self.models + table.name + for table in self.Base.metadata.sorted_tables + if table.name in self.models ] logger.debug("Unsorted models: %s", model_names) model_count = len(model_names) @@ -155,7 +168,7 @@ def sort_models(self): sort_round = 0 while swapped: sort_round += 1 - logger.debug('Sorting round: %d (%s)', sort_round, model_names) + logger.debug("Sorting round: %d (%s)", sort_round, model_names) sorted_models = [] for i in range(model_count): @@ -194,7 +207,7 @@ def create_model_table(self, model): model.name, message="failed to create in-memory table.", orig_exc=exc, - context=self.error_context + context=self.error_context, ) def load_model_data(self, path, model): @@ -209,45 +222,45 @@ def load_model_data(self, path, model): def load_model_data_collection(self, path, model): for full_filename in glob.iglob(os.path.join(path, "*.yml")): - # full_filename = os.path.join(path, '_all.yml') + # full_filename = os.path.join(path, '_all.yml') self.error_context.update(filename=full_filename) db_model = globals()[model.name] # load the collection data from the collection file - with open(full_filename, mode='rt', encoding=self.encoding) as f: + with open(full_filename, mode="rt", encoding=self.encoding) as f: collection = yaml.load(f.read(), Loader=yaml.FullLoader) if not isinstance(collection, list): raise InvalidModelCollectionDataError( - model.name, - context=self.error_context + model.name, context=self.error_context ) seen_entries = set() - logger.debug("Loading %d instance(s) for model: %s", len(collection), model.name) + logger.debug( + "Loading %d instance(s) for model: %s", len(collection), model.name + ) for item in collection: - if not isinstance(item, dict) or 'pk' not in item: + if not isinstance(item, dict) or "pk" not in item: raise InvalidModelCollectionDataError( - model.name, - context=self.error_context + model.name, context=self.error_context ) entry = StatikDatabaseInstance( - name=item['pk'], + name=item["pk"], from_dict=item, model=model, session=self.session, encoding=self.encoding, - markdown_config=self.markdown_config + markdown_config=self.markdown_config, ) # duplicate primary key! - if entry.field_values['pk'] in seen_entries: + if entry.field_values["pk"] in seen_entries: raise DuplicateModelInstanceError( model.name, - pk=entry.field_values['pk'], - context=self.error_context + pk=entry.field_values["pk"], + context=self.error_context, ) else: - seen_entries.add(entry.field_values['pk']) + seen_entries.add(entry.field_values["pk"]) try: db_entry = db_model(**entry.field_values) @@ -255,20 +268,22 @@ def load_model_data_collection(self, path, model): except Exception as exc: raise DataError( model.name, - pk=entry.field_values['pk'], + pk=entry.field_values["pk"], message="failed to insert entry into in-memory database.", orig_exc=exc, - context=self.error_context + context=self.error_context, ) - + self.error_context.clear() def load_model_data_from_files(self, path, model): db_model = globals()[model.name] - entry_files = list_files(path, ['yml', 'yaml', 'md']) - entry_files = [f for f in entry_files if not(f.endswith(".yml"))] + entry_files = list_files(path, ["yml", "yaml", "md"]) + entry_files = [f for f in entry_files if not (f.endswith(".yml"))] seen_entries = set() - logger.debug("Loading %d instance(s) for model: %s", len(entry_files), model.name) + logger.debug( + "Loading %d instance(s) for model: %s", len(entry_files), model.name + ) for entry_file in entry_files: entry = StatikDatabaseInstance( filename=os.path.join(path, entry_file), @@ -276,17 +291,15 @@ def load_model_data_from_files(self, path, model): session=self.session, encoding=self.encoding, markdown_config=self.markdown_config, - error_context=self.error_context + error_context=self.error_context, ) # duplicate primary key! - if entry.field_values['pk'] in seen_entries: + if entry.field_values["pk"] in seen_entries: raise DuplicateModelInstanceError( - model.name, - pk=entry.field_values['pk'], - context=self.error_context + model.name, pk=entry.field_values["pk"], context=self.error_context ) else: - seen_entries.add(entry.field_values['pk']) + seen_entries.add(entry.field_values["pk"]) try: db_entry = db_model(**entry.field_values) @@ -294,10 +307,10 @@ def load_model_data_from_files(self, path, model): except Exception as exc: raise DataError( model.name, - pk=entry.field_values['pk'], + pk=entry.field_values["pk"], message="failed to insert entry into in-memory database.", orig_exc=exc, - context=self.error_context + context=self.error_context, ) self.error_context.clear() @@ -319,13 +332,15 @@ def query(self, query, additional_locals=None, safe_mode=False): logger.debug("Attempting to execute database query: %s", query) if safe_mode and not isinstance(query, dict): - raise SafetyViolationError( - context=self.error_context - ) + raise SafetyViolationError(context=self.error_context) if isinstance(query, dict): logger.debug("Executing query in safe mode (MLAlchemy)") - return mlalchemy.parse_query(query).to_sqlalchemy(self.session, self.tables).all() + return ( + mlalchemy.parse_query(query) + .to_sqlalchemy(self.session, self.tables) + .all() + ) else: logger.debug("Executing unsafe query (Python exec())") if additional_locals is not None: @@ -333,15 +348,11 @@ def query(self, query, additional_locals=None, safe_mode=False): locals()[k] = v exec( - compile( - 'result = %s' % query.strip(), - '', - 'exec' - ), + compile("result = %s" % query.strip(), "", "exec"), globals(), - locals() + locals(), ) - return locals()['result'] + return locals()["result"] def shutdown(self): """Shuts down the database engine.""" @@ -352,10 +363,10 @@ def shutdown(self): class StatikDatabaseInstance(ContentLoadable): - def __init__(self, model=None, session=None, **kwargs): - super(StatikDatabaseInstance, self).__init__(content_field = model.content_field, - **kwargs) + super(StatikDatabaseInstance, self).__init__( + content_field=model.content_field, **kwargs + ) if model is None: raise MissingParameterError("model", context=self.error_context) self.model = model @@ -368,51 +379,69 @@ def __init__(self, model=None, session=None, **kwargs): # convert the vars to their underscored representation self.field_values = underscore_var_names(self.vars) - self.field_values['pk'] = self.name + self.field_values["pk"] = self.name # run through the foreign key fields to check their assignment for field_name in self.model.field_names: field = self.model.fields[field_name] - if isinstance(field, StatikDateTimeField) and \ - isinstance(self.field_values.get(field_name), basestring): + if isinstance(field, StatikDateTimeField) and isinstance( + self.field_values.get(field_name), basestring + ): # attempt to perform an intelligent date/time parse operation - self.field_values[field_name] = dateutil_parse(self.field_values[field_name]) + self.field_values[field_name] = dateutil_parse( + self.field_values[field_name] + ) # if it's a foreign key elif isinstance(field, StatikForeignKeyField): # if we've got a pk value for a foreign key field if field_name in self.field_values: - self.field_values['%s_id' % field_name] = self.field_values[field_name] + self.field_values["%s_id" % field_name] = self.field_values[ + field_name + ] del self.field_values[field_name] - elif isinstance(field, StatikManyToManyField) and (field_name in self.field_values): + elif isinstance(field, StatikManyToManyField) and ( + field_name in self.field_values + ): if not isinstance(self.field_values[field_name], list): raise InvalidFieldTypeError( self.model.name, field_name, "a list", - context=self.error_context + context=self.error_context, ) logger.debug( - "Attempting to look up primary keys for ManyToMany " + - "field relationship: %s", self.field_values[field_name] + "Attempting to look up primary keys for ManyToMany " + + "field relationship: %s", + self.field_values[field_name], ) - duplicates_in_array = find_duplicates_in_array(self.field_values[field_name]) + duplicates_in_array = find_duplicates_in_array( + self.field_values[field_name] + ) if duplicates_in_array: - logger.warning("Duplicates found in %s: %s (field: %s)", - self.filename, - duplicates_in_array, - field_name) - self.field_values[field_name] = list(set(self.field_values[field_name])) + logger.warning( + "Duplicates found in %s: %s (field: %s)", + self.filename, + duplicates_in_array, + field_name, + ) + self.field_values[field_name] = list( + set(self.field_values[field_name]) + ) # check if non-string items are present for item in self.field_values[field_name]: - if not isinstance(item, ("".__class__, u"".__class__)): - logger.warning("Non-string values found in array " + - "(field: %s, instance: %s, model: %s): %s", - field_name, self.field_values['pk'], self.model.name, item) - + if not isinstance(item, ("".__class__, "".__class__)): + logger.warning( + "Non-string values found in array " + + "(field: %s, instance: %s, model: %s): %s", + field_name, + self.field_values["pk"], + self.model.name, + item, + ) # convert the list of field values to a query to look up the # primary keys of the corresponding table @@ -422,64 +451,66 @@ def __init__(self, model=None, session=None, **kwargs): for item in self.field_values[field_name]: try: - self.session.query( - other_model - ).filter( + self.session.query(other_model).filter( other_model.pk == item ).one() except NoResultFound: # Only allow implicit tables if the model # has no fields other than 'pk' if len(other_model.__table__._columns) == 1: - missing_items.append({'pk': item}) + missing_items.append({"pk": item}) else: - logger.warning('%s not found in %s' % - (item, other_model)) + logger.warning("%s not found in %s" % (item, other_model)) else: - logger.debug('%s found', item) + logger.debug("%s found", item) - self.implicit_data_items.append( - (field.field_type, missing_items) - ) + self.implicit_data_items.append((field.field_type, missing_items)) original_values = self.field_values[field_name] - self.field_values[field_name] = self.session.query( - other_model - ).filter( - other_model.pk.in_(self.field_values[field_name]) - ).all() + self.field_values[field_name] = ( + self.session.query(other_model) + .filter(other_model.pk.in_(self.field_values[field_name])) + .all() + ) # Ensure that values appear in original order self.field_values[field_name].sort( - key=lambda x: original_values.index(x.pk)) + key=lambda x: original_values.index(x.pk) + ) # populate any Content field for this model if self.model.content_field is not None: self.field_values[self.model.content_field] = self.content - logger.debug('%s', self) + logger.debug("%s", self) def __repr__(self): result = ["StatikDatabaseInstance(model=%s" % self.model.name] for field_name, field_value in iteritems(self.field_values): model_field = self.model.fields.get(field_name, None) - if isinstance(model_field, StatikContentField) or isinstance(model_field, StatikTextField): + if isinstance(model_field, StatikContentField) or isinstance( + model_field, StatikTextField + ): result.append("%s=<...>" % field_name) else: result.append("%s=%s" % (field_name, field_value)) - result[-1] += ')' - return ', '.join(result) + result[-1] += ")" + return ", ".join(result) def __str__(self): return repr(self) def db_model_factory(Base, model, all_models): - def get_or_create_association_table(model1_name, model2_name): - _association_table_name = calculate_association_table_name(model1_name, model2_name) - logger.debug("Creating/getting ManyToMany relationship table: %s", _association_table_name) + _association_table_name = calculate_association_table_name( + model1_name, model2_name + ) + logger.debug( + "Creating/getting ManyToMany relationship table: %s", + _association_table_name, + ) if _association_table_name in globals(): return globals()[_association_table_name] @@ -487,35 +518,36 @@ def get_or_create_association_table(model1_name, model2_name): _association_table = Table( _association_table_name, Base.metadata, - Column('%s_pk' % model1_name.lower(), String, ForeignKey('%s.pk' % model1_name)), - Column('%s_pk' % model2_name.lower(), String, ForeignKey('%s.pk' % model2_name)) + Column( + "%s_pk" % model1_name.lower(), String, ForeignKey("%s.pk" % model1_name) + ), + Column( + "%s_pk" % model2_name.lower(), String, ForeignKey("%s.pk" % model2_name) + ), ) # track it in our globals set_global(_association_table_name, _association_table) return _association_table - logger.debug('-----') + logger.debug("-----") logger.debug("Generating model: %s", model.name) - model_fields = { - '__tablename__': model.name, - 'pk': Column(String, primary_key=True) - } + model_fields = {"__tablename__": model.name, "pk": Column(String, primary_key=True)} # populate all of the relevant additional relationships for this model for field_name, rel in iteritems(model.additional_rels): kwargs = {} - if rel.get('back_populates', None) is not None: - kwargs['back_populates'] = rel['back_populates'] - if rel.get('secondary', None) is not None: - kwargs['secondary'] = get_or_create_association_table(*rel['secondary']) + if rel.get("back_populates", None) is not None: + kwargs["back_populates"] = rel["back_populates"] + if rel.get("secondary", None) is not None: + kwargs["secondary"] = get_or_create_association_table(*rel["secondary"]) logger.debug( - 'Creating additional relationship %s.%s -> %s (%s)', + "Creating additional relationship %s.%s -> %s (%s)", model.name, field_name, - rel['to_model'], - kwargs + rel["to_model"], + kwargs, ) - model_fields[field_name] = relationship(rel['to_model'], **kwargs) + model_fields[field_name] = relationship(rel["to_model"], **kwargs) # now populate all of the standard fields for field_name in model.field_names: @@ -523,67 +555,63 @@ def get_or_create_association_table(model1_name, model2_name): if field.field_type in SQLALCHEMY_FIELD_MAPPER: # if it's a simple field model_fields[field.name] = Column( - field.name, - SQLALCHEMY_FIELD_MAPPER[field.field_type] + field.name, SQLALCHEMY_FIELD_MAPPER[field.field_type] ) elif field.field_type in all_models: # if it's a foreign key reference if isinstance(field, StatikForeignKeyField): - model_fields['%s_id' % field.name] = Column( - '%s_id' % field.name, - ForeignKey('%s.pk' % field.field_type) + model_fields["%s_id" % field.name] = Column( + "%s_id" % field.name, ForeignKey("%s.pk" % field.field_type) ) # if it's a self-referencing foreign key if field.field_type == model.name: - back_populates = field.back_populates or 'children' + back_populates = field.back_populates or "children" model_fields[back_populates] = relationship( field.field_type, - backref=backref(field_name, remote_side=[model_fields['pk']]) + backref=backref(field_name, remote_side=[model_fields["pk"]]), ) else: kwargs = {} if field.back_populates is not None: - kwargs['back_populates'] = field.back_populates - logger.debug('Field %s.%s has back-populates field name: %s', - model.name, field_name, field.back_populates + kwargs["back_populates"] = field.back_populates + logger.debug( + "Field %s.%s has back-populates field name: %s", + model.name, + field_name, + field.back_populates, ) else: - logger.debug('No back-populates field name for %s.%s', - model.name, field_name + logger.debug( + "No back-populates field name for %s.%s", + model.name, + field_name, ) - model_fields[field.name] = relationship( - field.field_type, - **kwargs - ) + model_fields[field.name] = relationship(field.field_type, **kwargs) elif isinstance(field, StatikManyToManyField): - association_table = get_or_create_association_table(model.name, field.field_type) + association_table = get_or_create_association_table( + model.name, field.field_type + ) - kwargs = {'secondary': association_table} + kwargs = {"secondary": association_table} if field.back_populates is not None: - kwargs['back_populates'] = field.back_populates + kwargs["back_populates"] = field.back_populates - logger.debug("Creating model ManyToMany field %s.%s -> %s (%s)", - model.name, field.name, field.field_type, kwargs - ) - model_fields[field.name] = relationship( + logger.debug( + "Creating model ManyToMany field %s.%s -> %s (%s)", + model.name, + field.name, field.field_type, - **kwargs + kwargs, ) + model_fields[field.name] = relationship(field.field_type, **kwargs) else: - raise InvalidFieldTypeError( - model.name, - field.name - ) + raise InvalidFieldTypeError(model.name, field.name) - Model = type( - str(model.name), - (Base,), - model_fields - ) + Model = type(str(model.name), (Base,), model_fields) logger.debug("Model %s fields = %s", model.name, model_fields) From 8cb68766ffc2a1dc35847bbe4c7b3c5984ab7d65 Mon Sep 17 00:00:00 2001 From: CLiu13 Date: Thu, 22 Nov 2018 14:08:14 -0500 Subject: [PATCH 36/38] database.py: Preserve default list value order This ensures that the default list value order will be the order in the resulting output. Fixes https://github.com/thanethomson/statik/issues/76 --- statik/database.py | 5 +++++ tests/modular/test_database.py | 15 ++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/statik/database.py b/statik/database.py index aabf606..d177414 100644 --- a/statik/database.py +++ b/statik/database.py @@ -479,6 +479,11 @@ def __init__(self, model=None, session=None, **kwargs): key=lambda x: original_values.index(x.pk) ) + # Ensure that values appear in original order + self.field_values[field_name].sort( + key=lambda x: original_values.index(x.pk) + ) + # populate any Content field for this model if self.model.content_field is not None: self.field_values[self.model.content_field] = self.content diff --git a/tests/modular/test_database.py b/tests/modular/test_database.py index ff9d93b..cad8a58 100644 --- a/tests/modular/test_database.py +++ b/tests/modular/test_database.py @@ -121,18 +121,15 @@ def test_database(self): room_tags = db.session.query(RoomTag).order_by(RoomTag.pk).all() self.assertEqual(5, len(room_tags)) - blueroom_tags = set([tag.pk for tag in guesthouse_rooms[0].tags]) + blueroom_tags = list([tag.pk for tag in guesthouse_rooms[0].tags]) self.assertEqual(4, len(blueroom_tags)) - self.assertIn('fireplace', blueroom_tags) - self.assertIn('double-bed', blueroom_tags) - self.assertIn('balcony', blueroom_tags) - self.assertIn('shower', blueroom_tags) + self.assertEqual( + ['fireplace', 'double-bed', 'balcony', 'shower'], blueroom_tags) - redroom_tags = set([tag.pk for tag in guesthouse_rooms[1].tags]) + redroom_tags = list([tag.pk for tag in guesthouse_rooms[1].tags]) self.assertEqual(3, len(redroom_tags)) - self.assertIn('fireplace', redroom_tags) - self.assertIn('single-bed', redroom_tags) - self.assertIn('shower', redroom_tags) + self.assertEqual( + ['fireplace', 'single-bed', 'shower'], redroom_tags) def assertInstanceEqual(self, expected, inst): for field_name, field_value in iteritems(expected): From 910a8a7c26a39c444b19504340fed2a065707119 Mon Sep 17 00:00:00 2001 From: Kai Chen Date: Tue, 4 Dec 2018 01:14:37 -0800 Subject: [PATCH 37/38] Allow access to variables defined in config in views Context defined in the config can now be accessed in views Closes https://github.com/thanethomson/statik/issues/54 --- statik/context.py | 6 +++--- tests/modular/test_context.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 tests/modular/test_context.py diff --git a/statik/context.py b/statik/context.py index 80eb86c..a9be54c 100644 --- a/statik/context.py +++ b/statik/context.py @@ -44,12 +44,12 @@ def __repr__(self): def __str__(self): return repr(self) - def build_dynamic(self, db, safe_mode=False): + def build_dynamic(self, db, extra=None, safe_mode=False): """Builds the dynamic context based on our current dynamic context entity and the given database.""" result = dict() for var, query in iteritems(self.dynamic): - result[var] = db.query(query, safe_mode=safe_mode) + result[var] = db.query(query, safe_mode=safe_mode, additional_locals=extra) return result def build_for_each(self, db, safe_mode=False, extra=None): @@ -68,7 +68,7 @@ def build(self, db=None, safe_mode=False, for_each_inst=None, extra=None): result = copy(self.initial) result.update(self.static) if self.dynamic: - result.update(self.build_dynamic(db, safe_mode=safe_mode)) + result.update(self.build_dynamic(db, extra=extra, safe_mode=safe_mode)) if self.for_each and for_each_inst: result.update(self.build_for_each(db, safe_mode=safe_mode, extra=extra)) if isinstance(extra, dict): diff --git a/tests/modular/test_context.py b/tests/modular/test_context.py new file mode 100644 index 0000000..ff3a0b4 --- /dev/null +++ b/tests/modular/test_context.py @@ -0,0 +1,16 @@ +# -*- coding:utf-8 -*- +import unittest + +from statik.context import StatikContext +from statik.database import StatikDatabase + + +class TestStatikContext(unittest.TestCase): + def test_build_context(self): + context = StatikContext(dynamic={'render_elm': 'True if my_var > 10 else False'}) + + result = context.build(db=StatikDatabase(models={}, data_path=''), extra={'my_var': 30}) + assert result.get('render_elm') is True + + result = context.build(db=StatikDatabase(models={}, data_path=''), extra={'my_var': 5}) + assert result.get('render_elm') is False From d37e02e345f3bd5215ac1d3501659a16f2e9bde6 Mon Sep 17 00:00:00 2001 From: Kai Chen Date: Sat, 8 Dec 2018 16:18:01 -0800 Subject: [PATCH 38/38] database.py: Allow for multiple foreign keys in same document Allow for multiple foreign keys in the same document by telling SQLAlchemy which foreign key to look for. Closes https://github.com/thanethomson/statik/issues/27 --- statik/database.py | 5 ++++- .../data_test_database/Address/adanac.yml | 2 ++ .../data_test_database/Address/washington.yml | 2 ++ .../Guest/gmerriweather.yml | 2 ++ .../data_test_database/Guest/manderson.yml | 2 ++ tests/modular/test_database.py | 19 ++++++++++++++++++- 6 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 tests/modular/data_test_database/Address/adanac.yml create mode 100644 tests/modular/data_test_database/Address/washington.yml diff --git a/statik/database.py b/statik/database.py index d177414..a3a33ae 100644 --- a/statik/database.py +++ b/statik/database.py @@ -593,7 +593,10 @@ def get_or_create_association_table(model1_name, model2_name): field_name, ) - model_fields[field.name] = relationship(field.field_type, **kwargs) + foreign_key = model_fields.get("%s_id" % field.name, None) + model_fields[field.name] = relationship( + field.field_type, foreign_keys=[foreign_key], **kwargs + ) elif isinstance(field, StatikManyToManyField): association_table = get_or_create_association_table( diff --git a/tests/modular/data_test_database/Address/adanac.yml b/tests/modular/data_test_database/Address/adanac.yml new file mode 100644 index 0000000..d3134e3 --- /dev/null +++ b/tests/modular/data_test_database/Address/adanac.yml @@ -0,0 +1,2 @@ +street: Adanac +postal_code: V5K 2S6 diff --git a/tests/modular/data_test_database/Address/washington.yml b/tests/modular/data_test_database/Address/washington.yml new file mode 100644 index 0000000..596c21b --- /dev/null +++ b/tests/modular/data_test_database/Address/washington.yml @@ -0,0 +1,2 @@ +street: Washington +postal_code: V5M 2H9 diff --git a/tests/modular/data_test_database/Guest/gmerriweather.yml b/tests/modular/data_test_database/Guest/gmerriweather.yml index 0ae320f..1e24188 100644 --- a/tests/modular/data_test_database/Guest/gmerriweather.yml +++ b/tests/modular/data_test_database/Guest/gmerriweather.yml @@ -1,3 +1,5 @@ first-name: Gary last-name: Merriweather email: gmerriweather@somewhere.com +home_address: adanac +business_address: washington diff --git a/tests/modular/data_test_database/Guest/manderson.yml b/tests/modular/data_test_database/Guest/manderson.yml index 1593680..87543e9 100644 --- a/tests/modular/data_test_database/Guest/manderson.yml +++ b/tests/modular/data_test_database/Guest/manderson.yml @@ -1,3 +1,5 @@ first-name: Michael last-name: Anderson email: manderson@somewhere.com +home_address: adanac +business_address: washington diff --git a/tests/modular/test_database.py b/tests/modular/test_database.py index cad8a58..5885754 100644 --- a/tests/modular/test_database.py +++ b/tests/modular/test_database.py @@ -10,9 +10,15 @@ from statik.models import * from statik.database import * +ADDRESS_MODEL = """street: String +postal_code: String +""" + GUEST_MODEL = """first-name: String last-name: String email: String +home_address: Address +business_address: Address """ GUESTHOUSE_MODEL = """guesthouse-name: String @@ -33,9 +39,10 @@ to-date: DateTime """ -MOCK_MODEL_NAMES = ['Guest', 'Guesthouse', 'GuesthouseRoom', 'Booking', 'RoomTag'] +MOCK_MODEL_NAMES = ['Address', 'Guest', 'Guesthouse', 'GuesthouseRoom', 'Booking', 'RoomTag'] MOCK_MODELS = { + 'Address': StatikModel(name='Address', from_string=ADDRESS_MODEL, model_names=MOCK_MODEL_NAMES), 'Guest': StatikModel(name='Guest', from_string=GUEST_MODEL, model_names=MOCK_MODEL_NAMES), 'Guesthouse': StatikModel(name='Guesthouse', from_string=GUESTHOUSE_MODEL, model_names=MOCK_MODEL_NAMES), 'GuesthouseRoom': StatikModel(name='GuesthouseRoom', from_string=GUESTHOUSE_ROOM_MODEL, model_names=MOCK_MODEL_NAMES), @@ -57,12 +64,14 @@ def setUp(self): def test_database(self): data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data_test_database') db = StatikDatabase(data_path, MOCK_MODELS) + Address = db.tables['Address'] Guest = db.tables['Guest'] Guesthouse = db.tables['Guesthouse'] GuesthouseRoom = db.tables['GuesthouseRoom'] Booking = db.tables['Booking'] RoomTag = db.tables['RoomTag'] + addresses = db.session.query(Address).order_by(Address.street).all() guests = db.session.query(Guest).order_by(Guest.last_name).all() self.assertEqual(2, len(guests)) self.assertInstanceEqual({ @@ -70,12 +79,20 @@ def test_database(self): 'last_name': 'Anderson', 'pk': 'manderson', 'email': 'manderson@somewhere.com', + 'home_address': addresses[0], + 'home_address_id': addresses[0].pk, + 'business_address': addresses[1], + 'business_address_id': addresses[1].pk, }, guests[0]) self.assertInstanceEqual({ 'first_name': 'Gary', 'last_name': 'Merriweather', 'pk': 'gmerriweather', 'email': 'gmerriweather@somewhere.com', + 'home_address': addresses[0], + 'home_address_id': addresses[0].pk, + 'business_address': addresses[1], + 'business_address_id': addresses[1].pk, }, guests[1]) guesthouses = db.session.query(Guesthouse).order_by(Guesthouse.guesthouse_name).all()