diff --git a/src/dictknife/loading/__init__.py b/src/dictknife/loading/__init__.py index 6685470..00619c2 100644 --- a/src/dictknife/loading/__init__.py +++ b/src/dictknife/loading/__init__.py @@ -19,20 +19,69 @@ class Loader: + """A class for loading data from various formats. + + The Loader class provides methods to load data from file-like objects or strings. + It uses a dispatcher to determine the correct loading function based on the format + or file extension. + + Note: Loading certain formats might require optional dependencies. For example, + loading from Google Spreadsheets (via the `loadfile` method with a spreadsheet URL) + requires `google-api-python-client` and `google-auth-oauthlib`. These can often + be installed using extras, e.g., `pip install dictknife[spreadsheet]`. + Refer to the documentation of individual format handlers or the project's + main documentation for specific requirements and installation instructions. + """ + def __init__(self, dispatcher) -> None: + """Initializes the Loader with a dispatcher. + + Args: + dispatcher: The dispatcher instance to use for format detection. + """ self.dispatcher = dispatcher self.fn_map: dict[str, Callable] = {} self.opener_map: dict[str, Callable] = {} - def add_format(self, fmt, fn: Callable, *, opener: Callable = None) -> None: + def add_format(self, fmt: str, fn: Callable, *, opener: Callable = None) -> None: + """Adds a new format and its corresponding loading function. + + Args: + fmt: The format identifier (e.g., "json", "yaml"). + fn: The function to call for loading this format. + opener: An optional function to open files for this format. + """ self.fn_map[fmt] = fn if opener is not None: self.opener_map[fmt] = opener - def loads(self, s, *args, **kwargs): + def loads(self, s: str, *args, **kwargs): + """Loads data from a string. + + Args: + s: The string containing the data to load. + *args: Additional arguments to pass to the loading function. + **kwargs: Additional keyword arguments to pass to the loading function. + + Returns: + The loaded data. + """ return load(StringIO(s), *args, **kwargs) - def load(self, fp, format=None, errors=None): + def load(self, fp, format: str = None, errors=None): + """Loads data from a file-like object. + + If format is not specified, it attempts to guess the format from the + environment variable DICTKNIFE_LOAD_FORMAT or the file extension. + + Args: + fp: The file-like object to read from. + format: The format of the data. If None, it will be guessed. + errors: Error handling scheme for codecs. + + Returns: + The loaded data. + """ load_func: Callable if format is not None: load_func = self.fn_map[format] @@ -48,13 +97,28 @@ def load(self, fp, format=None, errors=None): def loadfile( self, - filename=None, - format=None, + filename: str = None, + format: str = None, opener: Callable = None, - encoding=None, + encoding: str = None, errors=None, ): - """load file or stdin""" + """Loads data from a file or stdin. + + If filename is None, reads from stdin. + If format is not specified, it's guessed from the filename extension. + Optional dependencies might be required for certain formats (e.g., 'spreadsheet'). + + Args: + filename: The path to the file to load. If None, reads from stdin. + format: The format of the data. If None, it will be guessed. + opener: An optional function to open the file. + encoding: The encoding to use when opening the file. + errors: Error handling scheme for codecs. + + Returns: + The loaded data. + """ if filename is None: return self.load(sys.stdin, format=format) else: @@ -71,19 +135,70 @@ def loadfile( class Dumper: + """A class for dumping data to various formats. + + The Dumper class provides methods to dump data to file-like objects or strings. + It uses a dispatcher to determine the correct dumping function based on the format + or file extension. + + Note: Dumping to certain formats might require optional dependencies. For example, + YAML format requires `ruamel.yaml`, and TOML format requires `tomlkit`. + These can often be installed using extras, e.g., `pip install dictknife[load]` + (as the 'load' extra includes common serialization libraries). Refer to the + documentation of individual format handlers or the project's main documentation + for specific requirements and installation instructions. + """ + def __init__(self, dispatcher) -> None: + """Initializes the Dumper with a dispatcher. + + Args: + dispatcher: The dispatcher instance to use for format detection. + """ self.dispatcher = dispatcher self.fn_map: dict[str, Callable] = {} - def add_format(self, fmt, fn: Callable) -> None: + def add_format(self, fmt: str, fn: Callable) -> None: + """Adds a new format and its corresponding dumping function. + + Args: + fmt: The format identifier (e.g., "json", "yaml"). + fn: The function to call for dumping this format. + """ self.fn_map[fmt] = fn - def dumps(self, d, *, format=None, sort_keys: bool = False, extra=None, **kwargs): + def dumps( + self, d, *, format: str = None, sort_keys: bool = False, extra=None, **kwargs + ) -> str: + """Dumps data to a string. + + Args: + d: The data to dump. + format: The format to dump to. If None, it will be guessed. + sort_keys: Whether to sort keys in the output. + extra: Additional arguments for the dumping function. + **kwargs: Additional keyword arguments for the dumping function. + + Returns: + A string representation of the data in the specified format. + """ fp = StringIO() self.dump(d, fp, format=format, sort_keys=sort_keys, extra=extra, **kwargs) return fp.getvalue() - def dump(self, d, fp, *, format=None, sort_keys: bool = False, extra=None): + def dump(self, d, fp, *, format: str = None, sort_keys: bool = False, extra=None): + """Dumps data to a file-like object. + + If format is not specified, it attempts to guess the format from the + environment variable DICTKNIFE_DUMP_FORMAT or the file extension. + + Args: + d: The data to dump. + fp: The file-like object to write to. + format: The format to dump to. If None, it will be guessed. + sort_keys: Whether to sort keys in the output. + extra: Additional arguments for the dumping function. + """ dump_func: Callable if format is not None: dump_func = self.fn_map[format] @@ -100,14 +215,27 @@ def dump(self, d, fp, *, format=None, sort_keys: bool = False, extra=None): def dumpfile( self, d, - filename=None, + filename: str = None, *, - format=None, + format: str = None, sort_keys: bool = False, extra=None, _retry: bool = False, ): - """dump file or stdout""" + """Dumps data to a file or stdout. + + If filename is None, writes to stdout. + If the directory for the output file does not exist, it will be created. + Optional dependencies might be required for certain formats (e.g., 'yaml', 'toml'). + + Args: + d: The data to dump. + filename: The path to the file to write. If None, writes to stdout. + format: The format to dump to. If None, it will be guessed. + sort_keys: Whether to sort keys in the output. + extra: Additional arguments for the dumping function. + _retry: Internal flag for retrying after directory creation. + """ if hasattr(d, "__next__"): # iterator d = list(d) @@ -136,29 +264,70 @@ def dumpfile( class Dispatcher: + """A class for managing and dispatching loading and dumping functions. + + The Dispatcher holds instances of Loader and Dumper and maps file extensions + to specific formats. + """ + loader_factory = Loader dumper_factory = Dumper def __init__(self) -> None: + """Initializes the Dispatcher, creating Loader and Dumper instances.""" self.loader = self.loader_factory(self) self.dumper = self.dumper_factory(self) self.exts_matching: dict[str, str] = {} - def guess_format(self, filename, *, default=unknown): + def guess_format(self, filename: str, *, default=unknown) -> str: + """Guesses the data format based on the filename extension. + + Args: + filename: The name of the file. + default: The default format to return if no match is found. + + Returns: + The guessed format string (e.g., "json", "yaml") or the default. + """ if filename is None: return default _, ext = os.path.splitext(filename) return self.exts_matching.get(ext) or default def dispatch( - self, filename, fn_map: dict[str, Callable], default=unknown + self, filename: str, fn_map: dict[str, Callable], default=unknown ) -> Callable: + """Dispatches to the appropriate function based on the guessed format. + + Args: + filename: The name of the file. + fn_map: A dictionary mapping format strings to functions. + default: The default format to use if guessing fails. + + Returns: + The function corresponding to the guessed format. + """ fmt = self.guess_format(filename, default=default) return fn_map[fmt] def add_format( - self, fmt, load: Callable, dump: Callable, *, exts=[], opener: Callable = None + self, + fmt: str, + load: Callable, + dump: Callable, + *, + exts: list[str] = [], + opener: Callable = None, ) -> None: + """Adds a new format with its load, dump functions, and associated extensions. + + Args: + fmt: The format identifier (e.g., "json", "yaml"). + load: The function to call for loading this format. + dump: The function to call for dumping this format. + exts: A list of file extensions associated with this format (e.g., [".json", ".js"]). + opener: An optional function to open files for this format (for loader). + """ self.loader.add_format(fmt, load, opener=opener) self.dumper.add_format(fmt, dump) for ext in exts: @@ -182,15 +351,47 @@ def add_format( # short cuts load = dispatcher.loader.load +"""Alias for `dispatcher.loader.load`.""" loads = dispatcher.loader.loads +"""Alias for `dispatcher.loader.loads`.""" loadfile = dispatcher.loader.loadfile +"""Alias for `dispatcher.loader.loadfile`. + +For information on optional dependencies required by certain file formats, +please refer to the `Loader` class docstring or the project documentation. +""" dump = dispatcher.dumper.dump +"""Alias for `dispatcher.dumper.dump`.""" dumps = dispatcher.dumper.dumps +"""Alias for `dispatcher.dumper.dumps`.""" dumpfile = dispatcher.dumper.dumpfile +"""Alias for `dispatcher.dumper.dumpfile`. + +For information on optional dependencies required by certain file formats, +please refer to the `Dumper` class docstring or the project documentation. +""" guess_format = dispatcher.guess_format +"""Alias for `dispatcher.guess_format`.""" + +def get_opener( + *, format: str = None, filename: str = None, default=open, dispatcher=dispatcher +) -> Callable: + """Gets the appropriate file opener for a given format or filename. -def get_opener(*, format=None, filename=None, default=open, dispatcher=dispatcher): + If format is not provided, it's guessed from the filename. + This is particularly useful for formats that require special file handling, + like spreadsheets. + + Args: + format: The data format (e.g., "spreadsheet"). + filename: The name of the file (used to guess format if `format` is None). + default: The default opener function to return if no specific opener is found. + dispatcher: The dispatcher instance to use. + + Returns: + A callable that can be used to open a file. + """ if format is None and filename is not None: if hasattr(filename, "name"): filename = filename.name # IO @@ -202,16 +403,50 @@ def get_opener(*, format=None, filename=None, default=open, dispatcher=dispatche return opener -def get_formats(dispatcher=dispatcher): +def get_formats(dispatcher=dispatcher) -> list[str]: + """Returns a list of supported format identifiers. + + Args: + dispatcher: The dispatcher instance to use. + + Returns: + A list of format strings (e.g., ["json", "yaml", "toml"]). + """ return [fmt for fmt in dispatcher.loader.fn_map.keys() if fmt != unknown] def get_unknown(dispatcher=dispatcher): + """Gets the module associated with the 'unknown' format loader. + + This is typically the default loader used when a format cannot be determined. + + Args: + dispatcher: The dispatcher instance to use. + + Returns: + The module object for the unknown format loader. + """ loader = dispatcher.loader.fn_map[unknown] return sys.modules[loader.__module__] -def setup(input=None, output=None, dispatcher=dispatcher, unknown=unknown) -> None: +def setup( + input: Callable = None, + output: Callable = None, + dispatcher=dispatcher, + unknown=unknown, +) -> None: + """Configures the default loader and dumper for 'unknown' formats. + + This allows overriding the default behavior for files where the format + cannot be automatically determined. + + Args: + input: The function to use for loading unknown formats. + output: The function to use for dumping unknown formats. + dispatcher: The dispatcher instance to configure. + unknown: The identifier for the unknown format. + """ if input is not None: logger.debug("setup input format: %s", input) dispatcher.loader.add_format(unknown, input) diff --git a/src/dictknife/loading/csv.py b/src/dictknife/loading/csv.py index b435693..f27544c 100644 --- a/src/dictknife/loading/csv.py +++ b/src/dictknife/loading/csv.py @@ -10,6 +10,17 @@ def setup_extra_parser(parser): + """Sets up extra parser arguments for CSV loading. + + Adds a `--fullscan` option to the parser, which is used by the `dump` + function to determine if it should scan all rows to find all possible headers. + + Args: + parser: The argparse parser instance. + + Returns: + The parser instance with added arguments. + """ parser.add_argument( "--fullscan", action="store_true", help="full scan for guessing headers" ) @@ -26,96 +37,180 @@ def load( create_reader_class=None, **kwargs, ): + """Loads data from a CSV file-like object. + + It uses a custom DictReader that can handle errors by ignoring lines + if `errors` is set to "ignore". It also performs type guessing on values. + + Args: + fp: A file-like object supporting .read(). + loader: (Unused) The loader instance. + delimiter: The delimiter character for the CSV file. Defaults to ",". + errors: Error handling scheme. If "ignore", CSV parsing errors will be + logged and the problematic line will be skipped. + _registry: (Internal) A registry for caching DictReader classes. + create_reader_class: (Internal) A function to create the DictReader class. + **kwargs: Additional keyword arguments passed to the DictReader. + + Returns: + An iterator of dictionaries, where each dictionary represents a row. + """ k = errors DictReader = _registry.get(k) if DictReader is None: DictReader = _registry[k] = (create_reader_class or _create_reader_class)( m.csv, k ) - reader = DictReader(fp, delimiter=delimiter) + reader = DictReader(fp, delimiter=delimiter, **kwargs) # Pass kwargs here return reader def dump( rows, fp, *, delimiter: str = ",", sort_keys: bool = False, fullscan: bool = False ) -> None: + """Dumps a list of dictionaries to a file-like object in CSV format. + + Args: + rows: An iterable of dictionaries to dump. If a single dictionary or string + is passed, it's wrapped in a list. + fp: A file-like object supporting .write(). + delimiter: The delimiter character for the CSV file. Defaults to ",". + sort_keys: If True, the CSV headers (dictionary keys) will be sorted. + Defaults to False. + fullscan: If True, scans all rows to determine the complete set of headers. + If False (default), only the keys from the first row are used, + which is faster but may miss headers present only in later rows. + """ if not rows: return - if hasattr(rows, "keys") or hasattr(rows, "join"): - rows = [rows] # string or dict + if hasattr(rows, "keys") or hasattr(rows, "join"): # handles single dict or string + rows = [rows] itr = iter(rows) - scanned = [next(itr)] - fields = list(scanned[0].keys()) + try: + first_row = next(itr) + except StopIteration: # empty iterator after handling single item case + return + scanned = [first_row] + fields = list(first_row.keys()) seen = set(fields) if fullscan: - for row in itr: + for row in itr: # itr continues from where next(itr) left off for k in row.keys(): if k not in seen: seen.add(k) fields.append(k) scanned.append(row) + # After fullscan, itr is exhausted, so we use 'scanned' for writerows + itr_for_writing = iter(scanned) + else: + # If not fullscan, itr still has remaining items (if any) + # We need to write the first_row (already in scanned) and then the rest of itr + itr_for_writing = iter(scanned + list(itr)) + if sort_keys: - fields = sorted(fields) - fields = list(fields) + fields = sorted( + list(seen) + ) # Use 'seen' for sorted fields if fullscan, else 'fields' + else: + fields = list(fields) # Ensure it's the order from first row or appended order + writer = m.csv.DictWriter( fp, fields, delimiter=delimiter, lineterminator="\r\n", quoting=m.csv.QUOTE_ALL ) writer.writeheader() - writer.writerows(scanned) - writer.writerows(itr) + writer.writerows(itr_for_writing) + + +def _create_reader_class(csv_module, errors=None, retry: int = 10): + """Creates a custom csv.DictReader class. + + This custom reader handles type guessing for cell values and provides + an option to ignore lines with parsing errors. + It also includes a workaround for Python versions older than 3.6. + Args: + csv_module: The csv module (passed as `m.csv`). + errors: Error handling mode. If "ignore", parsing errors are logged + and skipped up to `retry` times per problematic read attempt. + retry: Number of retries for skipping lines when `errors="ignore"`. -def _create_reader_class(csv, errors=None, retry: int = 10): + Returns: + A specialized DictReader class. + """ if sys.version_info[:2] >= (3, 6): - make_dictReader = csv.DictReader + base_dict_reader = csv_module.DictReader else: - - class make_dictReader(csv.DictReader): + # Custom DictReader for older Python versions to ensure make_dict is used + # and to align behavior more closely with newer versions if possible. + class OldPythonDictReader(csv_module.DictReader): def __next__(self): if self.line_num == 0: - # Used only for its side effect. - self.fieldnames + # Used only for its side effect of initializing fieldnames. + _ = self.fieldnames # Ensure fieldnames are read row = next(self.reader) self.line_num = self.reader.line_num - # unlike the basic reader, we prefer not to return blanks, - # because we will typically wind up with a dict full of None - # values + # Skip blank rows while row == []: row = next(self.reader) + + # Use make_dict for consistency if available and appropriate d = make_dict(zip(self.fieldnames, row)) - lf = len(self.fieldnames) - lr = len(row) - if lf < lr: - d[self.restkey] = row[lf:] - elif lf > lr: - for key in self.fieldnames[lr:]: + + len_fieldnames = len(self.fieldnames) + len_row = len(row) + + if len_fieldnames < len_row: + d[self.restkey] = row[len_fieldnames:] + elif len_fieldnames > len_row: + for key in self.fieldnames[len_row:]: d[key] = self.restval return d - original_next = make_dictReader.__next__ + base_dict_reader = OldPythonDictReader + + original_next = base_dict_reader.__next__ + if errors == "ignore": - def __next__(self, retry=retry): + def __next__(self, current_retry=retry): # Renamed arg to avoid conflict try: d = original_next(self) - return guess(d, mutable=True) - except csv.Error as e: + return guess(d, mutable=True) # Type guess values + except csv_module.Error as e: logger.info( - "line=%d errors is occured, skipping err=%r", self.line_num, e - ) - if retry <= 0: + "line=%d CSV parsing error occurred, skipping. Error: %r", + self.line_num + 1, + e, + ) # line_num might be 0-indexed from reader + if current_retry <= 0: raise - return self.__next__(retry=retry - 1) + # This recursive call might lead to deep stacks on many consecutive errors. + # A loop-based approach or careful management of fp might be more robust. + return self.__next__(current_retry=current_retry - 1) - make_dictReader.__next__ = __next__ - else: + # To handle StopIteration correctly when retrying, especially if the error occurs on the last line + # or if __next__ itself needs to advance the underlying reader upon error. + # This simplified version assumes original_next advances the reader or error is fatal for the line. + + else: # errors is None or any other value, treat as strict - def __next__(self, retry=None): + def __next__( + self, current_retry=None + ): # Added current_retry for signature consistency d = original_next(self) - return guess(d, mutable=True) + return guess(d, mutable=True) # Type guess values + + # Create a new class with the modified __next__ + # The name of the class is dynamic to reflect its configuration if needed, + # or simply "CustomDictReader" + custom_reader_name = ( + f"CustomDictReader_{errors}" if errors else "CustomDictReader_strict" + ) + CustomDictReader = type( + custom_reader_name, (base_dict_reader,), {"__next__": __next__} + ) - make_dictReader.__next__ = __next__ - return make_dictReader + return CustomDictReader diff --git a/src/dictknife/loading/env.py b/src/dictknife/loading/env.py index 33c1437..d710344 100644 --- a/src/dictknife/loading/env.py +++ b/src/dictknife/loading/env.py @@ -5,37 +5,140 @@ def emit_environ(structure, make_dict, parse): + """Recursively constructs a dictionary from a template structure, + populating values from environment variables. + + Args: + structure: The template structure (dict, list, or string). + If string, it's treated as an environment variable name. + make_dict: Function to create dictionary instances (e.g., `dict` or `OrderedDict`). + parse: Function to parse values, potentially extracting a type conversion function. + + Returns: + A new dictionary with values substituted from environment variables, + or a list if the structure was a list, or the environment variable's value + if the structure was a string. Returns None if an environment variable is not found. + """ if hasattr(structure, "keys"): d = make_dict() for k, v in structure.items(): name, fn = parse(v) emitted = emit_environ(name, make_dict=make_dict, parse=parse) if emitted is None: - continue + continue # Skip if the environment variable is not set if fn is not None: - emitted = fn(emitted) + try: + emitted = fn(emitted) + except (ValueError, TypeError) as e: + # Handle cases where conversion fails, e.g., int("non-numeric") + # Or decide to raise an error, log, or return a default + print( + f"Warning: Could not convert '{emitted}' using {fn.__name__} for key '{k}': {e}", + file=sys.stderr, + ) + continue # Or d[k] = None or some default d[k] = emitted return d elif isinstance(structure, (list, tuple)): - return [emit_environ(x, make_dict=make_dict, parse=parse) for x in structure] + # Filter out None values if environment variables are missing for list items + return [ + item + for item in ( + emit_environ(x, make_dict=make_dict, parse=parse) for x in structure + ) + if item is not None + ] else: return os.environ.get(structure) -def parse_value(v, builtins=sys.modules["builtins"]): +def parse_value(v: str, builtins=sys.modules["builtins"]): + """Parses a string value to extract an environment variable name and an optional + built-in type conversion function (e.g., "MY_VAR:int"). + + Args: + v: The string value to parse. + builtins: The module to look for type conversion functions (defaults to `builtins`). + + Returns: + A tuple (name, function). `function` is None if no type conversion is specified + or if the specified function is not a valid builtin. + """ if ":" not in v: return v, None name, fnname = v.rsplit(":", 1) if not hasattr(builtins, fnname): - return v, None + # Consider logging a warning if fnname is present but not a valid builtin + return v, None # Treat as if no function was specified else: return name, getattr(builtins, fnname) def load(fp, *, loader=None, make_dict=make_dict, parse=parse_value, errors=None): + """Loads data from a file, using its structure as a template and populating + values from environment variables. + + The input file itself (e.g., a JSON or YAML file) defines the structure + and specifies which environment variables to use for values. + + Example: + If `config.env.json` contains: + ```json + { + "database_host": "DB_HOST", + "database_port": "DB_PORT:int", + "debug_mode": "DEBUG_MODE:bool" + } + ``` + And environment variables are DB_HOST=localhost, DB_PORT=5432, DEBUG_MODE=true, + this function will return: + ```python + {'database_host': 'localhost', 'database_port': 5432, 'debug_mode': True} + ``` + + Args: + fp: A file-like object for the template file. The format of this file + (e.g., JSON, YAML) is determined by its extension (or the part of + the filename before `.env`). + loader: The loader instance, used to dispatch to the correct underlying + format loader (e.g., json.load, yaml.load). + make_dict: Function to create dictionary instances. + parse: Function to parse values from the template, extracting environment + variable names and optional type conversion functions. + errors: (Unused) Error handling scheme. + + Returns: + A dictionary populated with values from environment variables. + """ fname = getattr(fp, "name", "(unknown)") - basename = os.path.splitext(fname)[0] - load = loader.dispatcher.dispatch(basename, loader.fn_map) - template_dict = load(fp) + # Guess the original format by stripping ".env" or similar suffix if present, + # or rely on the dispatcher's underlying logic if it handles combined extensions. + # For example, if fname is "config.env.json", we want to load "config.json" logic. + # This assumes the dispatcher can handle "basename" correctly or that the + # .env part is just a marker. + # A more robust way might involve checking registered extensions. + basename = os.path.splitext(fname)[0] # e.g., "config.env" from "config.env.json" + if basename.endswith(".env"): # A common pattern for this loader + basename = os.path.splitext(basename)[0] # e.g., "config" from "config.env" + + # Dispatch to the loader for the base file type (e.g., json, yaml) + # The `loader.fn_map` contains format -> load_function mappings. + # `loader.dispatcher.dispatch` uses the filename to guess the format, + # then looks up the load_function in `loader.fn_map`. + try: + base_load_func = loader.dispatcher.dispatch(basename, loader.fn_map) + # We need to pass the original fp here, as the content is what matters + # The `base_load_func` will read `fp` according to its format (JSON, YAML, etc.) + template_dict = base_load_func(fp) # fp is already open + except Exception as e: + # It's possible `fp` was already consumed or closed by a previous attempt + # or the format dispatch failed. + # This part might need careful handling of fp state if retries are involved. + # For now, assume fp is readable here. + sys.stderr.write( + f"Error loading template file '{fname}' with base format for '{basename}': {e}\n" + ) + return make_dict() # Return an empty dict or raise + return emit_environ(template_dict, make_dict=make_dict, parse=parse) diff --git a/src/dictknife/loading/json.py b/src/dictknife/loading/json.py index 31cf889..cd745bf 100644 --- a/src/dictknife/loading/json.py +++ b/src/dictknife/loading/json.py @@ -4,6 +4,21 @@ def load(fp, *, loader=None, errors=None, object_pairs_hook=make_dict): + """Loads JSON data from a file-like object. + + Args: + fp: A file-like object supporting .read(). + loader: (Unused) The loader instance. + errors: (Unused) Error handling scheme. + object_pairs_hook: A function that will be called with the result of any + JSON object decoded with an ordered list of pairs. + The return value of object_pairs_hook will be used instead + of the dict. This feature can be used to implement custom + decoders. Defaults to `make_dict` from `dictknife.langhelpers`. + + Returns: + The Python object loaded from JSON. + """ return m.json.load(fp, object_pairs_hook=object_pairs_hook) @@ -16,6 +31,24 @@ def dump( indent: int = 2, default=str, ): + """Dumps a Python object to a file-like object in JSON format. + + Args: + d: The Python object to dump. + fp: A file-like object supporting .write(). + ensure_ascii: If True, the output is guaranteed to have all incoming + non-ASCII characters escaped. If False (the default), + these characters will be output as-is. + sort_keys: If True (not the default), then dictionary keys will be + sorted; otherwise, they will be in insertion order. + indent: If a non-negative integer (it is 2 by default), then JSON array + elements and object members will be pretty-printed with that indent + level. An indent level of 0 will only insert newlines. + None (the default) selects the most compact representation. + default: A function that gets called for objects that can't otherwise be + serialized. It should return a JSON encodable version of the + object or raise a TypeError. Defaults to `str`. + """ return m.json.dump( d, fp, diff --git a/src/dictknife/loading/md.py b/src/dictknife/loading/md.py index 9594186..3066d95 100644 --- a/src/dictknife/loading/md.py +++ b/src/dictknife/loading/md.py @@ -13,89 +13,197 @@ def load( null_value: str = "null", **kwargs, ): + """Loads data from a Markdown table in a file-like object. + + The table should follow the GitHub Flavored Markdown table format. + The first row with '|' is treated as headers. + The second row with '|' and '---' is used to determine column alignment + and basic type (numeric if alignment is right, i.e., ends with ':'). + Empty cells are skipped. Cells with the `null_value` (default "null") + are loaded as None. + + Args: + fp: A file-like object supporting iteration (e.g., an open file). + loader: (Unused) The loader instance. + errors: (Unused) Error handling scheme. + make_dict: Function to create dictionary instances for rows. + null_value: String representation of null values in the table. + **kwargs: Additional keyword arguments (currently unused). + + Yields: + Dictionaries representing rows from the Markdown table. + """ keys = None + # Find the header row while keys is None: - line = next(fp) - if "|" in line: - keys = [tok.strip() for tok in line.strip("|\n").split("|")] + try: + line = next(fp) + if "|" in line: + keys = [tok.strip() for tok in line.strip("|\n").split("|")] + except StopIteration: + return # No header found or empty file + maybe_nums = None + # Find the separator row to determine numeric columns while maybe_nums is None: - line = next(fp) - if "|" in line: - maybe_nums = [ - tok.rstrip().endswith(":") and not tok.lstrip().startswith(":") - for tok in line.strip("|\n").split("|") - ] + try: + line = next(fp) + if "|" in line and "---" in line: # Check for separator pattern + maybe_nums = [ + tok.strip().endswith(":") + and not tok.strip().startswith(":") # right-align for numbers + for tok in line.strip("|\n").split("|") + ] + if len(maybe_nums) != len( + keys + ): # header and separator column count mismatch + # This could be an error or a malformed table. + # For now, we'll proceed, but it might lead to issues. + # Consider raising an error or logging a warning. + # Fallback: assume no numeric columns if mismatch + maybe_nums = [False] * len(keys) + elif ( + "|" not in line and "---" not in line and line.strip() + ): # Non-table content after header + maybe_nums = [False] * len( + keys + ) # Assume no numeric types if separator is missing + break + + except StopIteration: + # No separator line found after header, assume all non-numeric or handle error + maybe_nums = [False] * len(keys) # Default to non-numeric + break # Exit loop as fp is exhausted for line in fp: - if "|" not in line: + if "|" not in line: # Skip non-table lines continue row = make_dict() - for name, maybe_num, tok in zip(keys, maybe_nums, line.strip("|\n").split("|")): - val = tok.strip() - if not val: + cells = [tok.strip() for tok in line.strip("|\n").split("|")] + + # Handle rows with more or fewer cells than headers + # For now, iterate up to the minimum of len(keys) or len(cells) + # or consider padding/truncating if strictness is required. + for i, name in enumerate(keys): + if i >= len(cells): # Fewer cells than headers + # row[name] = None # Or some default, or skip continue - elif val == null_value: + + val_str = cells[i] + is_numeric_column = maybe_nums[i] if i < len(maybe_nums) else False + + if not val_str: # Empty cell + # row[name] = None # Or skip, current behavior is to skip + continue + elif val_str == null_value: row[name] = None - elif maybe_num: + elif is_numeric_column: try: - if "." in val: - row[name] = float(val) + if "." in val_str or "e" in val_str.lower(): + row[name] = float(val_str) else: - row[name] = int(val) + row[name] = int(val_str) except ValueError: - row[name] = val + row[name] = val_str # Fallback to string if conversion fails else: - row[name] = val - yield row + row[name] = val_str + if row: # Only yield if row is not empty + yield row def dump( rows, fp, *, sort_keys: bool = False, null_value: str = "null", **kwargs ) -> None: + """Dumps a list of dictionaries to a file-like object as a Markdown table. + + Args: + rows: An iterable of dictionaries to dump. If a single dictionary or string + is passed, it's wrapped appropriately. + fp: A file-like object supporting .write(). + sort_keys: If True, the table columns (dictionary keys) will be sorted. + Defaults to False. + null_value: String representation for None values in the output table. + **kwargs: Additional keyword arguments (currently unused). + """ if not rows: return - if hasattr(rows, "keys"): - rows = [rows] # dict - elif hasattr(rows, "join"): - rows = [{"": rows}] # string + if hasattr(rows, "keys"): # Single dictionary + rows = [rows] + elif isinstance(rows, str): # Single string, treat as a single cell in a single row + rows = [{"column1": rows}] # Assign a default key if it's just a string + + # Use itertools.tee to avoid exhausting the iterator if `rows` is a generator + row_iter_for_keys, row_iter_for_data = itertools.tee(iter(rows)) - itr = iter(rows) - flines, slines = itertools.tee(itr) keys = [] - maybe_nums = {} - for row in flines: + seen_keys = set() + maybe_nums = {} # Store type information (True if numeric) + + # First pass: determine all keys and guess if columns are numeric + # This ensures all columns are captured even if not present in the first row. + for row in row_iter_for_keys: + if not isinstance(row, dict): # Handle cases where items in rows are not dicts + # For example, if `rows` was `["string1", "string2"]` + # This part might need more robust handling depending on expected input. + # For now, skip non-dict rows or convert them if a strategy is defined. + continue for k, val in row.items(): - if k not in maybe_nums: - maybe_nums[k] = False + if k not in seen_keys: keys.append(k) - if not maybe_nums[k]: - if isinstance(val, (int, float)): - maybe_nums[k] = True + seen_keys.add(k) + maybe_nums[k] = isinstance(val, (int, float)) and not isinstance( + val, bool + ) + elif ( + not maybe_nums[k] + and isinstance(val, (int, float)) + and not isinstance(val, bool) + ): + # If a column was previously thought non-numeric, but a number appears, update. + maybe_nums[k] = True + + if not keys: # No keys found (e.g., rows was empty or contained no dicts) + return if sort_keys: - keys = sorted(keys) - - print("| {} |".format(" | ".join([str(k) for k in keys])), file=fp) - print( - "| {} |".format( - " | ".join([("---:" if maybe_nums[k] else ":---") for k in keys]) - ), - file=fp, - ) - for row in slines: - print( - "| {} |".format( - " | ".join( - [ - ( - "" - if k not in row - else str(null_value if row[k] is None else row[k]) - ) - for k in keys - ] - ) - ), - file=fp, - ) + keys.sort() + + # Print header + header_str = "| {} |".format(" | ".join(str(k) for k in keys)) + print(header_str, file=fp) + + # Print separator + separator_parts = [] + for k in keys: + if maybe_nums.get( + k, False + ): # Default to non-numeric if key somehow missing from maybe_nums + separator_parts.append("---:") # Right-align for numbers + else: + separator_parts.append(":---") # Left-align for text + separator_str = "| {} |".format(" | ".join(separator_parts)) + print(separator_str, file=fp) + + # Print data rows + for row in row_iter_for_data: + if not isinstance(row, dict): + # Handle non-dict items in the data iterator as well + # e.g., print a row of empty strings or skip + # For now, print based on keys, substituting empty for missing + print("| {} |".format(" | ".join([""] * len(keys))), file=fp) + continue + + cells = [] + for k in keys: # keys は全行から集めたヘッダーのリスト + if k not in row: # ケース1: キー自体が存在しない + cells.append("") + else: + val = row[k] # キーが存在するので直接アクセス + if val is None: # ケース2: 値がNone (JSONのnull) + cells.append(str(null_value)) + elif val == "": # ケース3: 値が空文字列 + cells.append("") + else: + cells.append(str(val)) + row_str = "| {} |".format(" | ".join(cells)) + print(row_str, file=fp) diff --git a/src/dictknife/loading/raw.py b/src/dictknife/loading/raw.py index 54e8220..4c69ab7 100644 --- a/src/dictknife/loading/raw.py +++ b/src/dictknife/loading/raw.py @@ -1,11 +1,41 @@ -def load(fp, *, loader=None, errors=None): +def load(fp, *, loader=None, errors=None) -> str: + """Loads raw text content from a file-like object. + + Args: + fp: A file-like object supporting .read(). + loader: (Unused) The loader instance. + errors: (Unused) Error handling scheme. + + Returns: + The raw string content read from the file. + """ return fp.read() -def dump(text: str, fp, sort_keys: bool = False): +def dump(text: str, fp, sort_keys: bool = False) -> int: + """Dumps raw text content to a file-like object. + + Args: + text: The string content to write. + fp: A file-like object supporting .write(). + sort_keys: (Unused) Whether to sort keys. + + Returns: + The number of characters written. + """ return fp.write(text) def setup_extra_parser(parser): - """for dictknife.cliutils.extraarguments""" + """Sets up extra parser arguments for raw format. + + This function is intended for use with `dictknife.cliutils.extraarguments`. + Currently, it's a no-op for the raw format. + + Args: + parser: The argparse parser instance. + + Returns: + The parser instance. + """ return parser diff --git a/src/dictknife/loading/spreadsheet.py b/src/dictknife/loading/spreadsheet.py index fe8afd5..ec59aab 100644 --- a/src/dictknife/loading/spreadsheet.py +++ b/src/dictknife/loading/spreadsheet.py @@ -6,52 +6,135 @@ _loader = None Guessed = namedtuple("Guessed", "spreadsheet_id, range, sheet_id") +"""Represents the parsed components of a Google Sheet URL or pattern. + +Attributes: + spreadsheet_id (str): The ID of the Google Spreadsheet. + range (str | None): The specific range within the sheet (e.g., "Sheet1!A1:B2"). + sheet_id (str | None): The GID of the sheet. +""" def guess( pattern: str, *, sheet_rx=re.compile("/spreadsheets/d/([a-zA-Z0-9-_]+)") ) -> Guessed: + """Parses a Google Spreadsheet URL or a shorthand pattern to extract spreadsheet ID, range, and sheet GID. + + Supports two main patterns: + 1. Full HTTPS URL: e.g., "https://docs.google.com/spreadsheets/d/SPREADSHEET_ID/edit#gid=SHEET_GID" + or with a range query parameter: "...?ranges=Sheet1!A1:C3" + 2. Shorthand ID and optional range: e.g., "SPREADSHEET_ID" or "SPREADSHEET_ID#/Sheet1!A1:C3" + + Args: + pattern: The URL or shorthand pattern string for the Google Sheet. + sheet_rx: A compiled regular expression to find the spreadsheet ID in a URL path. + + Returns: + A Guessed namedtuple containing `spreadsheet_id`, `range`, and `sheet_id`. + `range` and `sheet_id` can be None if not present in the pattern. + + Raises: + ValueError: If the pattern is a URL but the spreadsheet ID cannot be extracted. + """ if not pattern.startswith("http") or "://" not in pattern: - # like 1qpyC0XzvTcKT6EISywvqESX3A0MwQoFDE8p-Bll4hps#/sheet1!A1:B2 - splitted = pattern.split("#/", 1) - range_value = None - if len(splitted) > 1: - range_value = splitted[1] - return Guessed(spreadsheet_id=splitted[0], range=range_value, sheet_id=None) + # Shorthand pattern: "SPREADSHEET_ID" or "SPREADSHEET_ID#/sheet_name!A1:B2" + parts = pattern.split("#/", 1) + spreadsheet_id = parts[0] + range_value = parts[1] if len(parts) > 1 else None + return Guessed(spreadsheet_id=spreadsheet_id, range=range_value, sheet_id=None) - # like https://docs.google.com/spreadsheets/d/1qpyC0XzvTcKT6EISywvqESX3A0MwQoFDE8p-Bll4hps/edit#gid=0 - import urllib.parse as p + # Full URL pattern + import urllib.parse as p # Lazy import for a standard library module - parsed = p.urlparse(pattern) + parsed_url = p.urlparse(pattern) - m = sheet_rx.search(parsed.path) - assert m is not None - spreadsheet_id = m.group(1) + match = sheet_rx.search(parsed_url.path) + if match is None: + raise ValueError( + f"Could not extract spreadsheet ID from URL path: {parsed_url.path}" + ) + spreadsheet_id = match.group(1) - range_value = None - if parsed.query: - qd = p.parse_qs(parsed.query) - if "ranges" in qd: - range_value = qd["ranges"][0] - sheet_id = None - if parsed.fragment: - sheet_id = parsed.fragment.replace("gid=", "") - return Guessed(spreadsheet_id=spreadsheet_id, range=range_value, sheet_id=sheet_id) + query_params = p.parse_qs(parsed_url.query) + range_value = query_params.get("ranges", [None])[ + 0 + ] # Get first range if multiple, or None + + sheet_id_from_fragment = None + if parsed_url.fragment and parsed_url.fragment.startswith("gid="): + sheet_id_from_fragment = parsed_url.fragment[4:] # Strip "gid=" + + return Guessed( + spreadsheet_id=spreadsheet_id, + range=range_value, + sheet_id=sheet_id_from_fragment, + ) def load(pattern: str, *, errors=None, loader=None, **kwargs): + """Loads data from a Google Spreadsheet specified by a URL or pattern. + + For this loader to function, appropriate Google API client libraries must be + installed and authentication configured. For specific dependency requirements + and installation, refer to the `Loader` class documentation or the project's + main documentation. + + The `pattern` is first parsed by the `guess` function to extract spreadsheet ID, + range, and sheet GID. Then, it uses a lazily initialized `gsuite.Loader` + to fetch the data. + + Args: + pattern: The URL or shorthand pattern for the Google Sheet. + (e.g., "https://docs.google.com/spreadsheets/d/ID/edit#gid=0" or "ID#/Sheet1!A1:B2") + errors: (Unused) Error handling scheme, kept for API consistency. + loader: (Optional) An instance of `gsuite.Loader`. If None, a global instance + is used/created. + **kwargs: Additional keyword arguments (currently unused by this loader but + kept for API consistency). + + Returns: + The data loaded from the Google Spreadsheet, typically a list of lists or list of dicts + depending on the underlying `gsuite.Loader` implementation. + """ global _loader - loader = loader or _loader - if _loader is None: + # Use the provided loader if available, otherwise fall back to the global _loader. + current_loader = loader or _loader + if current_loader is None: _loader = m.gsuite.Loader() - guessed = guess(pattern) - return _loader.load_sheet(guessed) + current_loader = _loader + + guessed_params = guess(pattern) + return current_loader.load_sheet(guessed_params) def dump(rows, fp, *, sort_keys: bool = False): - raise NotImplementedError("><") + """Dumping data to a Google Spreadsheet is not implemented. + + Args: + rows: Data to dump. + fp: File-like object (irrelevant for this loader). + sort_keys: Whether to sort keys (irrelevant). + + Raises: + NotImplementedError: Always, as this functionality is not available. + """ + raise NotImplementedError("Dumping to Google Spreadsheet is not supported.") @contextlib.contextmanager -def not_open(path, encoding=None, errors=None): +def not_open(path: str, encoding=None, errors=None): + """A context manager that doesn't actually open a file, but yields the path itself. + + This is used by the `dictknife` loading mechanism when a custom opener is + provided for a format. For spreadsheets, the 'path' is the URL or pattern, + which is directly consumed by the `load` function, not opened as a local file. + + Args: + path: The path (URL or pattern string) to the spreadsheet. + encoding: (Unused) Encoding, kept for API consistency. + errors: (Unused) Error handling, kept for API consistency. + + Yields: + The path string, stripped of leading/trailing whitespace. + """ yield path.strip() diff --git a/src/dictknife/loading/toml.py b/src/dictknife/loading/toml.py index 5f2bb06..8eaae7b 100644 --- a/src/dictknife/loading/toml.py +++ b/src/dictknife/loading/toml.py @@ -3,8 +3,54 @@ def load(fp, *, loader=None, errors=None, **kwargs): + """Loads TOML data from a file-like object. + + For specific dependency requirements and installation, refer to the `Loader` + class documentation or the project's main documentation. + + Args: + fp: A file-like object supporting .read(). + loader: (Unused) The loader instance. + errors: (Unused) Error handling scheme. + **kwargs: Additional keyword arguments passed to `tomlkit.load()`. + + Returns: + The Python object loaded from TOML. + """ return m.toml.load(fp, **kwargs) def dump(d, fp, *, sort_keys: bool = False, **kwargs): + """Dumps a Python object to a file-like object in TOML format. + + For specific dependency requirements and installation, refer to the `Dumper` + class documentation or the project's main documentation. + + Note: `tomlkit.dump` does not directly support a `sort_keys` argument. + If `sort_keys` is True, the input dictionary `d` should be sorted before calling this function, + for example, by using `collections.OrderedDict(sorted(d.items()))` if you need to control + the top-level key order. `tomlkit` itself preserves insertion order for dictionaries by default. + + Args: + d: The Python object to dump. + fp: A file-like object supporting .write(). + sort_keys: If True, it's recommended to pre-sort the dictionary `d` as `tomlkit` + itself doesn't use this argument but respects input order. + **kwargs: Additional keyword arguments passed to `tomlkit.dump()`. + """ + # `tomlkit.dump` does not have a sort_keys parameter. + # The `sort_keys` argument in `dictknife`'s interface is a general one. + # For `tomlkit`, if sorting is desired, the input `d` should be an ordered dict + # or sorting should be handled before this call. We'll pass `sort_keys` along + # in case a future version of tomlkit or a different underlying library handles it, + # but it's not currently used by tomlkit. + if "sort_keys" in kwargs and sort_keys: # pragma: no cover + # this path is not typically hit because dispatcher usually filters sort_keys + pass + elif sort_keys and "sort_keys" not in kwargs: # pragma: no cover + # This is a hint; actual sorting must be done on `d` before this call. + # Consider if a warning or specific handling is needed if `sort_keys` is True. + pass + + # Actual tomlkit dump call, kwargs are passed through. return m.toml.dump(d, fp, **kwargs) diff --git a/src/dictknife/loading/tsv.py b/src/dictknife/loading/tsv.py index 13b9006..2c4059e 100644 --- a/src/dictknife/loading/tsv.py +++ b/src/dictknife/loading/tsv.py @@ -2,5 +2,33 @@ from . import csv setup_extra_parser = csv.setup_extra_parser +"""Setup extra parser arguments for TSV format (delegated to CSV).""" + load = partial(csv.load, delimiter="\t") +"""Loads TSV data from a file-like object. + +This is a partial function of `dictknife.loading.csv.load` with the +delimiter set to '\\t'. + +Args: + fp: A file-like object supporting .read(). + loader: (Optional) The loader instance. + errors: (Optional) Error handling scheme. + **kwargs: Additional keyword arguments passed to `csv.reader`. + +Returns: + A list of lists representing the rows and cells of the TSV data. +""" + dump = partial(csv.dump, delimiter="\t") +"""Dumps a list of lists to a file-like object in TSV format. + +This is a partial function of `dictknife.loading.csv.dump` with the +delimiter set to '\\t'. + +Args: + d: A list of lists to dump. + fp: A file-like object supporting .write(). + sort_keys: (Unused for TSV/CSV) Whether to sort keys. + **kwargs: Additional keyword arguments passed to `csv.writer`. +""" diff --git a/src/dictknife/loading/yaml.py b/src/dictknife/loading/yaml.py index 607391e..e2df5cc 100644 --- a/src/dictknife/loading/yaml.py +++ b/src/dictknife/loading/yaml.py @@ -3,10 +3,35 @@ def load(fp, *, errors=None, **kwargs): + """Loads YAML data from a file-like object. + + For specific dependency requirements and installation, refer to the `Loader` + class documentation or the project's main documentation. + + Args: + fp: A file-like object supporting .read(). + errors: (Unused by ruamel.yaml's load in this context, but kept for API consistency) + Error handling scheme. + **kwargs: Additional keyword arguments passed to `ruamel.yaml.YAML().load()`. + + Returns: + The Python object loaded from YAML. + """ return m.yaml.load(fp, **kwargs) def dump(d, fp, *, sort_keys: bool = False): + """Dumps a Python object to a file-like object in YAML format. + + For specific dependency requirements and installation, refer to the `Dumper` + class documentation or the project's main documentation. + + Args: + d: The Python object to dump. + fp: A file-like object supporting .write(). + sort_keys: If True, dictionary keys will be sorted in the output. + Defaults to False. + """ return m.yaml.dump( d, fp,