From 7f21d1422602cce402a6c2702e81f5b4c9f1f2e7 Mon Sep 17 00:00:00 2001 From: mfyuce Date: Thu, 3 Dec 2020 15:33:20 +0300 Subject: [PATCH 1/2] dax support and fixes --- xmla/olap/xmla/connection.py | 14 +-- xmla/olap/xmla/formatreader.py | 151 ++++++++++++++++++++++++--------- xmla/olap/xmla/utils.py | 60 +++++++------ 3 files changed, 149 insertions(+), 76 deletions(-) diff --git a/xmla/olap/xmla/connection.py b/xmla/olap/xmla/connection.py index b105680..01f04ef 100644 --- a/xmla/olap/xmla/connection.py +++ b/xmla/olap/xmla/connection.py @@ -9,7 +9,7 @@ from zeep.transports import Transport #import types -from .formatreader import TupleFormatReader +from .formatreader import TupleFormatReader, DAXFormatReader from .utils import * import logging @@ -165,15 +165,17 @@ def Execute(self, command, dimformat="Multidimensional", axisFormat="TupleFormat", **kwargs): if isinstance(command, stringtypes): command=as_etree({"Statement": command}) - props = {"Format":dimformat, "AxisFormat":axisFormat} + props = {"Format":dimformat, "AxisFormat": axisFormat} props.update(kwargs) - plist = as_etree({"PropertyList":props}) + plist = as_etree({"PropertyList": props}) try: - res = self.service.Execute(Command=command, Properties=plist, _soapheaders=self._soapheaders) - root = res.body["return"]["_value_1"] - return TupleFormatReader(fromETree(root, ns=schema_xmla_mddataset)) + root_raw = res.body["return"]["_value_1"] + if root_raw.tag.startswith("{{{}}}".format(schema_xmla_rowset)): + return DAXFormatReader(root_raw, fromETree(root_raw, ns=schema_xmla_rowset)) + else: + return TupleFormatReader(root_raw) except Fault as fault: raise XMLAException(fault.message, dictify(fromETree(fault.detail, ns=None))) diff --git a/xmla/olap/xmla/formatreader.py b/xmla/olap/xmla/formatreader.py index ced5e56..dcdd0a5 100644 --- a/xmla/olap/xmla/formatreader.py +++ b/xmla/olap/xmla/formatreader.py @@ -2,6 +2,17 @@ import types from olap.interfaces import IMDXResult import zope.interface +from collections import namedtuple +from zeep import ns + +nsmap = { + "soap": ns.SOAP_11, + "soap-env": ns.SOAP_ENV_11, + "wsdl": ns.WSDL, + "xsd": ns.XSD, + "sql": "urn:schemas-microsoft-com:xml-sql" +} + @zope.interface.implementer(IMDXResult) class TupleFormatReader(object): @@ -9,20 +20,21 @@ class TupleFormatReader(object): def __init__(self, tupleresult): self.root = tupleresult self.cellmap = self.mapOrdinalsToCells() - + def mapOrdinalsToCells(self): "Return a dict mapping ordinals to cells" m = {} # "getattr" for the case where there are no cells # aslist if there is only one cell - for cell in aslist(getattr(self.root.CellData, "Cell", [])): - m[int(cell._CellOrdinal)] = cell - + if self.root.get("CellData"): + for cell in aslist(getattr(self.root.CellData, "Cell", [])): + m[int(cell._CellOrdinal)] = cell + return m - + def getCellByOrdinal(self, ordinal): return self.cellmap.get(ordinal, {}) - + def getAxisTuple(self, axis): """Returns the tuple on axis with name , usually 'Axis0', 'Axis1', 'SlicerAxis'. If axis is a number return tuples on the -th axis. @@ -31,7 +43,7 @@ def getAxisTuple(self, axis): res = None try: if isinstance(axis, stringtypes): - ax = [x for x in aslist(self.root.Axes.Axis) if x._name == axis][0] + ax = [x for x in aslist(self.root.Axes.Axis) if x._name == axis][0] else: ax = aslist(self.root.Axes.Axis)[axis] res = [] @@ -40,7 +52,7 @@ def getAxisTuple(self, axis): except AttributeError: pass return res - + def getSlice(self, properties=None, **kw): """ getSlice(property=None [,Axis=n|Axis=[i1,i2,..,ix]]) @@ -69,21 +81,21 @@ def getSlice(self, properties=None, **kw): result.getSlice(properties=["Value", "FmtValue"]) """ - axisranges = [] # list per axis the element indices to include - - #n.b: this assumes, axis are listed from Axis0,...AxisN in the ExecuteResponse, - #otherwise the ordinal values would be useless anyway - + axisranges = [] # list per axis the element indices to include + + # n.b: this assumes, axis are listed from Axis0,...AxisN in the ExecuteResponse, + # otherwise the ordinal values would be useless anyway + # at this offset we find the first requested index of the dimension - firstindexoffset=2 - hyperelemcount=1 - - axlist= aslist(getattr(self.root.Axes, "Axis", [])) + firstindexoffset = 2 + hyperelemcount = 1 + + axlist = aslist(getattr(self.root.Axes, "Axis", [])) # filter out possible SlicerAxis axlist = [ax for ax in axlist if ax._name != "SlicerAxis"] - + for ax in axlist: - + if ax._name in kw: # only include listed indices indexrange = kw[ax._name] @@ -92,16 +104,16 @@ def getSlice(self, properties=None, **kw): # are the tupleindices valid? maxtups = len(getattr(ax.Tuples, "Tuple", [])) - toolarge=[idx for idx in indexrange if idx >= maxtups or idx < 0] + toolarge = [idx for idx in indexrange if idx >= maxtups or idx < 0] if toolarge: raise ValueError( "The tuple requested do not exist on axis '%s': %s" % \ - (ax._name, indexrange)) - + (ax._name, indexrange)) + else: # include all possible indices - indexrange=list(range(len(getattr(ax.Tuples, "Tuple", [])))) - + indexrange = list(range(len(getattr(ax.Tuples, "Tuple", [])))) + if not indexrange: # we have requested an empty set from an axis # by calling sth like this: getSlice(Axis2=[]) @@ -115,7 +127,7 @@ def getSlice(self, properties=None, **kw): # or more generally: # (#Axes - #EmptyAxes) == dim(result) (not counting SlicerAxis) return [] - + # first element is a helper to calc the ordinal value from a cell's coord, # second is the iteration index indexrange = [hyperelemcount * x for x in indexrange] @@ -123,49 +135,104 @@ def getSlice(self, properties=None, **kw): axisranges.append(axisrange) # hyperelemcount for the n-th Axis is the number # of cells in the subcube spanned by Axis(0)..Axis(n-1) - hyperelemcount = hyperelemcount*len(ax.Tuples.Tuple) + hyperelemcount = hyperelemcount * len(ax.Tuples.Tuple) # add an entry for the slicer axisranges.append([firstindexoffset, [], 0]) - + lastdimchange = 0 while lastdimchange < len(axisranges): # calc ordinal number of cell ordinal = 0 for axisrange in axisranges: - hyperelemcount=axisrange[axisrange[0]] + hyperelemcount = axisrange[axisrange[0]] ordinal = ordinal + hyperelemcount - + cell = self.getCellByOrdinal(ordinal) if properties is None: axisranges[0][1].append(cell) else: if isinstance(properties, stringtypes): - d = getattr(cell, properties, - None) + d = getattr(cell, properties, + None) else: d = {} - for prop in aslist(properties): - d[prop] = getattr(cell, prop, - None) + for prop in aslist(properties): + d[prop] = getattr(cell, prop, + None) axisranges[0][1].append(d) - + # advance to next requested element in slice - lastdimchange=0 + lastdimchange = 0 while lastdimchange < len(axisranges): axisrange = axisranges[lastdimchange] - if axisrange[0] < len(axisrange)-1: - axisrange[0] = axisrange[0]+1 + if axisrange[0] < len(axisrange) - 1: + axisrange[0] = axisrange[0] + 1 break else: axisrange[0] = firstindexoffset - - lastdimchange = lastdimchange+1 + + lastdimchange = lastdimchange + 1 if lastdimchange < len(axisranges): axisranges[lastdimchange][1].append(axisrange[1]) axisrange[1] = [] - + # as the last dimension is the sliceraxis it has only one member, # so we can safely unpack the first element # in that element our resulting multidimensional array has been accumulated - return axisranges[lastdimchange-1][1][0] + return axisranges[lastdimchange - 1][1][0] + + +@zope.interface.implementer(IMDXResult) +class DAXFormatReader(object): + + def __init__(self, root_raw, root): + self.root = root + self.root_raw = root_raw + self.res_raw = getattr(root, "row", []) + if self.res_raw: + self.res = aslist(self.res_raw) + self.rows = [] + self.description = self._get_description() + self._arrangeData() + + def _get_description(self): + """ + Return description from a single row. + + We only return the name, type (inferred from the data) and if the values + can be NULL. String columns in Druid are NULLable. Numeric columns are NOT + NULL. + """ + ret = [] + res = self.root_raw.findall("xsd:schema/xsd:complexType[@name='row']/xsd:sequence/xsd:element", + namespaces=nsmap) + + for i in range(len(res)): + c = res[i] + + t = c.attrib.get('type') + ret.append( + { + "name": c.attrib.get("{{{}}}field".format(nsmap["sql"])), + "type": t + } + ) + return ret + + def _arrangeData(self): + Row = None + for irow in range(len(self.res)): + row = self.res[irow] + + if Row is None: + keys = [d["name"] for d in self.description] + Row = namedtuple('Row', keys, rename=True) + + values = [] + + for key, value in row.items(): + if key != "text": + values.append(value) + self.rows.append(Row(*values)) + diff --git a/xmla/olap/xmla/utils.py b/xmla/olap/xmla/utils.py index ff614aa..d5c244a 100644 --- a/xmla/olap/xmla/utils.py +++ b/xmla/olap/xmla/utils.py @@ -140,33 +140,37 @@ def fromETree(e, ns): p = Data() nst = ns_name(ns, "*") valtype = ns_name(schema_instance, "type") - for (k,v) in e.attrib.items(): - setattr(p, "_"+k, v) - p.text = e.text - if p.text and p.text.strip() == "": + if e is not None: + for (k,v) in e.attrib.items(): + setattr(p, "_"+k, v) + p.text = e.text + if p.text and p.text.strip() == "": + p.text=None + else: p.text=None - if valtype in e.attrib: - if e.attrib[valtype] in ["xsd:int", "xsd:unsignedInt"]: - p.text = int(p.text) - delattr(p, "_"+valtype) - elif e.attrib[valtype] in ["xsd:long"]: - p.text = int(p.text) if six.PY3 else long(p.text) - delattr(p, "_"+valtype) - elif e.attrib[valtype] in ["xsd:double", "xsd:float"]: - p.text = float(p.text) - delattr(p, "_"+valtype) - for c in e.findall(nst): - t = QName(c) - - cd = fromETree(c, ns) - #if len(cd.__dict__) == 1: - if len(cd) == 1: - cd = cd.text - v = getattr(p, t.localname, None) - if v is not None: - if not isinstance(v, list): - setattr(p, t.localname, [v]) - getattr(p, t.localname).append(cd) - else: - setattr(p, t.localname, cd) + if e is not None: + if valtype in e.attrib: + if e.attrib[valtype] in ["xsd:int", "xsd:unsignedInt"]: + p.text = int(p.text) + delattr(p, "_"+valtype) + elif e.attrib[valtype] in ["xsd:long"]: + p.text = int(p.text) if six.PY3 else long(p.text) + delattr(p, "_"+valtype) + elif e.attrib[valtype] in ["xsd:double", "xsd:float"]: + p.text = float(p.text) + delattr(p, "_"+valtype) + for c in e.findall(nst): + t = QName(c) + + cd = fromETree(c, ns) + #if len(cd.__dict__) == 1: + if len(cd) == 1: + cd = cd.text + v = getattr(p, t.localname, None) + if v is not None: + if not isinstance(v, list): + setattr(p, t.localname, [v]) + getattr(p, t.localname).append(cd) + else: + setattr(p, t.localname, cd) return p \ No newline at end of file From e371f14e9bae61ef3b2a0db54d0c281d34ae7a4a Mon Sep 17 00:00:00 2001 From: Mehmet YUCE Date: Thu, 3 Dec 2020 17:51:02 +0300 Subject: [PATCH 2/2] Update connection.py --- xmla/olap/xmla/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xmla/olap/xmla/connection.py b/xmla/olap/xmla/connection.py index 01f04ef..78ea085 100644 --- a/xmla/olap/xmla/connection.py +++ b/xmla/olap/xmla/connection.py @@ -175,7 +175,7 @@ def Execute(self, command, dimformat="Multidimensional", if root_raw.tag.startswith("{{{}}}".format(schema_xmla_rowset)): return DAXFormatReader(root_raw, fromETree(root_raw, ns=schema_xmla_rowset)) else: - return TupleFormatReader(root_raw) + return TupleFormatReader(fromETree(root_raw, ns=schema_xmla_mddataset)) except Fault as fault: raise XMLAException(fault.message, dictify(fromETree(fault.detail, ns=None)))