Irods query on 4 2 stable

CMakeLists.txt

@@ -9,6 +9,10 @@ set(CMAKE_EXE_LINKER_FLAGS_INIT "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++")
 
 project(irods_rule_engine_plugin-python CXX)
 
+include(GenerateExportHeader)
+set(CMAKE_CXX_VISIBILITY_PRESET default)
+set(CMAKE_VISIBILITY_INLINES_HIDDEN 0)
+
 include(${IRODS_TARGETS_PATH})
 
 set(IRODS_PLUGIN_VERSION "${IRODS_VERSION}.${IRODS_PLUGIN_REVISION}")
@@ -90,7 +94,15 @@ set(
       irods_server
       )
 
-  target_compile_definitions(${PLUGIN} PRIVATE ${IRODS_RULE_ENGINE_PYTHON_PLUGIN_COMPILE_DEFINITIONS} ${IRODS_COMPILE_DEFINITIONS})
+  target_compile_definitions(${PLUGIN} PRIVATE ${IRODS_RULE_ENGINE_PYTHON_PLUGIN_COMPILE_DEFINITIONS} ${IRODS_COMPILE_DEFINITIONS}
+                                             RODS_SERVER
+                                             IRODS_QUERY_ENABLE_SERVER_SIDE_API
+                                             RODS_ENABLE_SYSLOG
+                                             IRODS_FILESYSTEM_ENABLE_SERVER_SIDE_API
+                                             IRODS_IO_TRANSPORT_ENABLE_SERVER_SIDE_API
+                                             SPDLOG_FMT_EXTERNAL
+                                             SPDLOG_NO_TLS
+  )
   #target_compile_options(${PLUGIN} PRIVATE -Wno-write-strings)
   set_property(TARGET ${PLUGIN} PROPERTY CXX_STANDARD ${IRODS_CXX_STANDARD})
 
@@ -116,6 +128,8 @@ install(
   ${CMAKE_SOURCE_DIR}/core.py.template
   ${CMAKE_SOURCE_DIR}/session_vars.py
   ${CMAKE_SOURCE_DIR}/genquery.py
+  ${CMAKE_SOURCE_DIR}/irods_query_wrapper.py
+  ${CMAKE_SOURCE_DIR}/test_irods_query.py
   DESTINATION etc/irods
   )
 

irods_query_wrapper.py

@@ -0,0 +1,74 @@
+import irods_query as _irods_query
+import re
+
+class IrodsQuery(object):
+
+    class Row( object ):
+        def keys(self): return self.columns
+        def as_dict(self): return { k : self[k] for k in self.keys() }
+        def __init__(self,values,columns):
+            self.values = values
+            self.columns = columns
+            self.columns_dict = dict(zip(columns,values))
+        def __iter__(self):
+            return iter(self.values)
+        def __getitem__(self,n):
+            if isinstance(n,int):   return self.values[n]
+            elif isinstance(n,str): return self.columns_dict[n]
+            else:                   raise KeyError
+
+    from irods_query import (query_iterator, vector_of_string)
+
+    GENERAL  = _irods_query.query_type.GENERAL
+    SPECIFIC = _irods_query.query_type.SPECIFIC
+
+    __attributes = re.split('\s+', 'comm zone_hint query_string auto_fields query_type query_limit row_offset bind_args')
+
+    def __init__(self, comm,             # rei.rsComm from rule parameter
+                       query_string,     # "select FIELD1,FIELD2 from ... where ... "
+                       auto_fields = (), # Up to n dictionary-like keys to correspond with FIELD(0..n-1)
+                       zone_hint = "",
+                       query_limit = 0,
+                       row_offset = 0,
+                       query_type = _irods_query.query_type.GENERAL, # IrodsQuery.GENERAL
+                       bind_args = () ): # args for ? ; only if query_type is SPECIFIC
+        self.__qi = None
+        for x in self.__attributes:
+            setattr(self,x,locals()[x])
+
+    def __iter__(self):
+        if self.__qi is None:
+            self.__args = self.vector_of_string()
+            self.__args.assign(self.bind_args)
+            self.__qi = iter( self.query_iterator( self.comm,
+                                                   self.query_string,
+                                                   self.__args, "", 0, 0, self.query_type))
+        return self
+
+    def next(self):
+        try:
+            n = next( self.__qi )
+        except StopIteration as e:
+            self.__qi = None
+            raise
+        else:
+            return self.Row(n, self.auto_fields)
+
+    @property 
+    def args(self): return tuple(self.__args)
+
+# -- usage in core.py --
+#
+# from irods_query_wrapper import ( IrodsQuery )
+# 
+# def gen_q(arg,cbk,rei):
+#     for (i,d,c) in IrodsQuery (rei.rsComm, "select DATA_ID,DATA_NAME,COLL_NAME"):
+#         cbk.writeLine ("stdout", "id = %s : %s/%s" % (i,c,d) )
+# 
+# def spec_q(arg,cbk,rei):
+#     for row in IrodsQuery( rei.rsComm, "listGroupsForUser",
+#                            bind_args=('dan',),
+#                            auto_fields=('grp_id','grp_name'), 
+#                            query_type = IrodsQuery.SPECIFIC
+#                          ):
+#         cbk.writeLine ("stdout", "Group ID {grp_id} NAME {grp_name}".format(**row))

irods_rule_engine_plugin-python.cxx

@@ -4,7 +4,9 @@
 #include <ctime>
 #include <fstream>
 #include <list>
+#include <vector>
 #include <string>
+#include <iterator>
 #include <memory>
 
 #include "boost/date_time.hpp"
@@ -14,6 +16,9 @@
 #include "boost/filesystem/operations.hpp"
 #include "boost/filesystem/operations.hpp"
 
+#define IRODS_QUERY_ENABLE_SERVER_SIDE_API 1
+#include "irods_query.hpp"
+
 #include "rodsErrorTable.h"
 #include "irods_error.hpp"
 #include "irods_re_plugin.hpp"
@@ -28,6 +33,9 @@
 
 #define register
 #include "boost/python/slice.hpp"
+#include "boost/python/iterator.hpp"
+#include "boost/python/stl_iterator.hpp"
+#include "boost/python/enum.hpp"
 #include "boost/python/module_init.hpp"
 #undef register
 
@@ -311,6 +319,48 @@ namespace
         }
     }; // struct CallbackWrapper
 
+
+    template<typename T>
+    void vector_assign(std::vector<T>& l, bp::object o) {
+        // Turn a Python sequence into an STL input range
+        bp::stl_input_iterator<T> begin(o), end;
+        l.assign(begin, end);
+    }
+
+    using Qiter = irods::query<rsComm_t>;
+    using Qtype = Qiter::query_type;
+
+    BOOST_PYTHON_MODULE(irods_query)
+    {
+      bp::enum_<Qtype> ("query_type")     .value("GENERAL", Qiter::GENERAL)
+                                          .value("SPECIFIC", Qiter::SPECIFIC)
+                                          .export_values();
+
+      bp::class_<Qiter,boost::noncopyable> qiterclass
+            ("query_iterator",bp::init<rsComm_t*            , // server comm handle
+                                       const std::string&   , // query string
+                                       const std::vector<std::string>*, // _specific_query_args
+                                       const std::string& ,   // _zone_hint (= "" default)
+                                       uintmax_t            , // query limit (= 0 default)
+                                       uintmax_t            , // row offset  (= 0 default)
+                                       Qiter::query_type>()); // query type (GENERAL = 0, SPECIFIC = 1)
+
+      qiterclass.def(bp::init<rsComm_t*,const std::string&>())  // either query type but without bind args
+      .def(bp::init<rsComm_t*            , // server comm handle
+                    const std::string&   , // query string,
+                    uintmax_t            , // query limit (= 0 default)
+                    uintmax_t            , // row offset  (= 0 default)
+                    Qiter::query_type>())  // query type (GENERAL = 0, SPECIFIC = 1)
+                .def("__iter__",bp::iterator<Qiter>());
+
+      using Vs = std::vector<std::string>;
+
+      bp::class_<std::vector<std::string> >("vector_of_string")
+      .def("assign", &vector_assign<std::string>)
+      .def("__getitem__",+[](const std::vector<std::string> &obj, int i) {return obj.at(i);} )
+      .def("__len__",&Vs::size);
+    }
+
     BOOST_PYTHON_MODULE(plugin_wrappers)
     {
         bp::class_<RuleCallWrapper>("RuleCallWrapper", bp::no_init)
@@ -335,11 +385,14 @@ irods::error start(irods::default_re_ctx&, const std::string& _instance_name)
         PyImport_AppendInittab("plugin_wrappers", &initplugin_wrappers);
         PyImport_AppendInittab("irods_types", &initirods_types);
         PyImport_AppendInittab("irods_errors", &initirods_errors);
+        PyImport_AppendInittab("irods_query", &initirods_query);
 #else
         PyImport_AppendInittab("plugin_wrappers", &PyInit_plugin_wrappers);
         PyImport_AppendInittab("irods_types", &PyInit_irods_types);
         PyImport_AppendInittab("irods_errors", &PyInit_irods_errors);
+        PyImport_AppendInittab("irods_query", &PyInit_irods_query);
 #endif
+
         Py_InitializeEx(0);
         std::lock_guard<std::recursive_mutex> lock{python_mutex};
         boost::filesystem::path full_path( boost::filesystem::current_path() );
@@ -355,6 +408,7 @@ irods::error start(irods::default_re_ctx&, const std::string& _instance_name)
         bp::object plugin_wrappers = bp::import("plugin_wrappers");
         bp::object irods_types = bp::import("irods_types");
         bp::object irods_errors = bp::import("irods_errors");
+        bp::object irods_query = bp::import("irods_query");
 
         StringFromPythonUnicode::register_converter();
     }
@@ -503,9 +557,12 @@ irods::error exec_rule(const irods::default_re_ctx&,
         bp::object core_namespace = core_module.attr("__dict__");
         bp::object irods_types = bp::import("irods_types");
         bp::object irods_errors = bp::import("irods_errors");
+        bp::object irods_query = bp::import("irods_query");
 
         core_namespace["irods_types"] = irods_types;
         core_namespace["irods_errors"] = irods_errors;
+        core_namespace["irods_types"] = irods_types;
+        core_namespace["irods_query"] = irods_query;
 
         bp::object rule_function = core_module.attr(rule_name.c_str());
 
@@ -635,13 +692,15 @@ irods::error exec_rule_text(const irods::default_re_ctx&,
             bp::object main_namespace = main_module.attr("__dict__");
             bp::object irods_types = bp::import("irods_types");
             bp::object irods_errors = bp::import("irods_errors");
+            bp::object irods_query = bp::import("irods_query");
 
             // Import global INPUT and OUTPUT variables
             main_namespace["global_vars"] = global_vars_python;
 
             // Import global constants
             main_namespace["irods_types"] = irods_types;
             main_namespace["irods_errors"] = irods_errors;
+            main_namespace["irods_query"] = irods_query;
 
             // Parse input rule_text into useable Python fcns
             // Delete first line ("@external")
@@ -744,6 +803,7 @@ irods::error exec_rule_expression(irods::default_re_ctx&,
         bp::object main_module = bp::import("__main__");
         bp::object irods_types = bp::import("irods_types");
         bp::object irods_errors= bp::import("irods_errors");
+        bp::object irods_query = bp::import("irods_query");
         bp::object main_namespace = main_module.attr("__dict__");
 
         // Import global INPUT and OUTPUT variables
@@ -752,6 +812,7 @@ irods::error exec_rule_expression(irods::default_re_ctx&,
         // Import globals
         main_namespace["irods_types"] = irods_types;
         main_namespace["irods_errors"] = irods_errors;
+        main_namespace["irods_query"] = irods_query;
 
         // Add def expressionFcn(rule_args, callback):\n to start of rule text
         std::string rule_name = "expressionFcn";

query_python_demo.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+case $1 in
+  0) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qc2gmain('1')" null ruleExecOut ;;
+  1) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qc2gmain('0')" null ruleExecOut ;;
+  2) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qc2dmain()" null ruleExecOut ;;
+  3) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qc1main()" null ruleExecOut ;;
+  4) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qc2main()" null ruleExecOut ;;
+  5) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qcmain()" null ruleExecOut ;;
+  6) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qmain()" null ruleExecOut ;;
+  7) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qqmain('2')" null ruleExecOut ;;
+  8) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qqmain('5')" null ruleExecOut ;;
+  9) irule -r irods_rule_engine_plugin-irods_rule_language-instance "qqmain('7')" null ruleExecOut ;;
+  *) exit 123;;
+esac