ConfigurePython.cxx

 
#include "Framework/ConfigurePython.h"
 
/*~~~~~~~~~~~~*/
/*   python   */
/*~~~~~~~~~~~~*/
#include "Python.h"
 
/*~~~~~~~~~~~~~~~~*/
/*   C++ StdLib   */
/*~~~~~~~~~~~~~~~~*/
#include <any>
#include <cstring>
#include <iostream>
#include <string>
#include <vector>
 
namespace framework {
 
std::string ConfigurePython::root_module = "ldmxcfg";
std::string ConfigurePython::root_class = "Process";
std::string ConfigurePython::root_object = "lastProcess";
 
static std::string getPyString(PyObject* pyObj) {
  std::string retval;
  PyObject* pyStr = PyUnicode_AsEncodedString(pyObj, "utf-8", "Error ~");
  retval = PyBytes_AS_STRING(pyStr);
  Py_XDECREF(pyStr);
  return retval;
}
 
std::string repr(PyObject* obj) {
  PyObject* py_repr = PyObject_Repr(obj);
  if (py_repr == nullptr) return "";
  std::string str = getPyString(py_repr);
  Py_XDECREF(py_repr);
  return str;
}
 
PyObject* extractDictionary(PyObject* obj) {
#if PY_MAJOR_VERSION != 3
#error("Framework requires compiling with Python3")
#else
#if PY_MINOR_VERSION != 10 && PY_MINOR_VERSION != 6
#warning("Unrecognized Python3 minor version. Unsure if accessing C API properly for configuration.")
#endif
  PyObject** p_dictionary{_PyObject_GetDictPtr(obj)};
  if (p_dictionary == NULL) {
    if (PyDict_Check(obj)) {
      return obj;
    } else {
      EXCEPTION_RAISE("ObjFail",
                      "Python Object '" + repr(obj) +
                          "' does not have __dict__ member and is not a dict.");
    }
  }
  return *p_dictionary;
#endif
}
 
static std::map<std::string, std::any> getMembers(PyObject* object) {
  PyObject* dictionary{extractDictionary(object)};
  PyObject *key(0), *value(0);
  Py_ssize_t pos = 0;
 
  std::map<std::string, std::any> params;
 
  while (PyDict_Next(dictionary, &pos, &key, &value)) {
    std::string skey{getPyString(key)};
    if (PyLong_Check(value)) {
      if (PyBool_Check(value)) {
        params[skey] = bool(PyLong_AsLong(value));
      } else {
        params[skey] = int(PyLong_AsLong(value));
      }
    } else if (PyFloat_Check(value)) {
      params[skey] = PyFloat_AsDouble(value);
    } else if (PyUnicode_Check(value)) {
      params[skey] = getPyString(value);
    } else if (PyList_Check(value)) {
      // assume everything is same value as first value
      if (PyList_Size(value) > 0) {
        auto vec0{PyList_GetItem(value, 0)};
 
        if (PyLong_Check(vec0)) {
          std::vector<int> vals;
 
          for (auto j{0}; j < PyList_Size(value); j++)
            vals.push_back(PyLong_AsLong(PyList_GetItem(value, j)));
 
          params[skey] = vals;
 
        } else if (PyFloat_Check(vec0)) {
          std::vector<double> vals;
 
          for (auto j{0}; j < PyList_Size(value); j++)
            vals.push_back(PyFloat_AsDouble(PyList_GetItem(value, j)));
 
          params[skey] = vals;
 
        } else if (PyUnicode_Check(vec0)) {
          std::vector<std::string> vals;
          for (Py_ssize_t j = 0; j < PyList_Size(value); j++) {
            PyObject* elem = PyList_GetItem(value, j);
            vals.push_back(getPyString(elem));
          }
 
          params[skey] = vals;
        } else if (PyList_Check(vec0)) {
          // a list in a list ??? oof-dah
          if (PyList_Size(vec0) > 0) {
            auto vecvec0{PyList_GetItem(vec0, 0)};
            if (PyLong_Check(vecvec0)) {
              std::vector<std::vector<int>> vals;
              for (auto j{0}; j < PyList_Size(value); j++) {
                auto subvec{PyList_GetItem(value, j)};
                std::vector<int> subvals;
                for (auto k{0}; k < PyList_Size(subvec); k++) {
                  subvals.push_back(PyLong_AsLong(PyList_GetItem(subvec, k)));
                }
                vals.push_back(subvals);
              }
              params[skey] = vals;
            } else if (PyFloat_Check(vecvec0)) {
              std::vector<std::vector<double>> vals;
              for (auto j{0}; j < PyList_Size(value); j++) {
                auto subvec{PyList_GetItem(value, j)};
                std::vector<double> subvals;
                for (auto k{0}; k < PyList_Size(subvec); k++) {
                  subvals.push_back(
                      PyFloat_AsDouble(PyList_GetItem(subvec, k)));
                }
                vals.push_back(subvals);
              }
              params[skey] = vals;
            } else if (PyUnicode_Check(vecvec0)) {
              std::vector<std::vector<std::string>> vals;
              for (auto j{0}; j < PyList_Size(value); j++) {
                auto subvec{PyList_GetItem(value, j)};
                std::vector<std::string> subvals;
                for (auto k{0}; k < PyList_Size(subvec); k++) {
                  subvals.push_back(getPyString(PyList_GetItem(subvec, k)));
                }
                vals.push_back(subvals);
              }
              params[skey] = vals;
            } else if (PyList_Check(vecvec0)) {
              EXCEPTION_RAISE("BadConf",
                              "A python list with dimension greater than 2 is "
                              "not supported.");
            } else {
              // RECURSION zoinks!
              std::vector<std::vector<framework::config::Parameters>> vals;
              for (auto j{0}; j < PyList_Size(value); j++) {
                auto subvec{PyList_GetItem(value, j)};
                std::vector<framework::config::Parameters> subvals;
                for (auto k{0}; k < PyList_Size(subvec); k++) {
                  subvals.emplace_back();
                  subvals.back().setParameters(
                      getMembers(PyList_GetItem(subvec, k)));
                }
                vals.push_back(subvals);
              }
              params[skey] = vals;
            }
          }  // non-zero size
        } else {
          // RECURSION zoinks!
          // If the objects stored in the list doesn't
          // satisfy any of the above conditions, just
          // create a vector of parameters objects
          std::vector<framework::config::Parameters> vals;
          for (auto j{0}; j < PyList_Size(value); ++j) {
            auto elem{PyList_GetItem(value, j)};
            vals.emplace_back();
            vals.back().setParameters(getMembers(elem));
          }
          params[skey] = vals;
        }  // type of object in python list
      }    // python list has non-zero size
    } else {
      // object got here, so we assume
      // it is a higher level object
      //(same logic as last option for a list)
 
      // RECURSION zoinks!
      framework::config::Parameters val;
      val.setParameters(getMembers(value));
 
      params[skey] = val;
 
    }  // python object type
  }    // loop through python dictionary
 
  return params;
}
 
ConfigurePython::ConfigurePython(const std::string& pythonScript, char* args[],
                                 int nargs) {
  // assumes that nargs >= 0
  //  this is true always because we error out if no python script has been
  //  found
 
  // load a handle to the config file into memory (and check that it exists)
  FILE* config_file{fopen(pythonScript.c_str(), "r")};
  if (config_file == NULL) {
    EXCEPTION_RAISE("ConfigDNE",
                    "Passed config script '" + pythonScript +
                        "' is not accessible.\n"
                        "    Did you make a typo in the path to the script?\n"
                        "    Are you referencing a directory that is not "
                        "mounted to the container?");
  }
 
  std::string cmd = pythonScript;
  if (pythonScript.rfind("/") != std::string::npos) {
    cmd = pythonScript.substr(pythonScript.rfind("/") + 1);
  }
  cmd = cmd.substr(0, cmd.find(".py"));
 
  // python needs the argument list as if you are on the command line
  //  targs = [ script , arg0 , arg1 , ... ] ==> len(targs) = nargs+1
  // PySys_SetArgvEx uses wchar_t instead of char in python3
  wchar_t** targs = new wchar_t*[nargs + 1];
  targs[0] = Py_DecodeLocale(pythonScript.c_str(), NULL);
  for (int i = 0; i < nargs; i++) targs[i + 1] = Py_DecodeLocale(args[i], NULL);
 
  // name our program after the script that is being run
  Py_SetProgramName(targs[0]);
 
  // start up python interpreter
  Py_Initialize();
 
  // The third argument to PySys_SetArgvEx tells python to import
  // the args and add the directory of the first argument to
  // the PYTHONPATH
  // This way, the command to import the module just needs to be
  // the name of the python script
  PySys_SetArgvEx(nargs + 1, targs, 1);
 
  // run the file as a python script
  if (PyRun_AnyFile(config_file, pythonScript.c_str()) != 0) {
    PyErr_Print();
    EXCEPTION_RAISE("ConfigureError", "Problem running python script.");
  }
 
  // script has been run so we can
  // free up arguments to python script
  fclose(config_file);
  for (int i = 0; i < nargs + 1; i++) PyMem_RawFree(targs[i]);
  delete[] targs;
 
  // when a script runs in python, the script itself becomes the module
  // named __main__, we retrieve a handle to that module by "importing"
  // it (since it is already within the python interpreter, we are just
  // getting the handle and not actually running anything here
  PyObject* script = PyImport_ImportModule("__main__");
  if (script == NULL) {
    PyErr_Print();
    EXCEPTION_RAISE("ConfigureError", "Problem loading python script");
  }
  PyObject* pCMod = PyObject_GetAttrString(script, root_module.c_str());
  Py_DECREF(script);  // don't need the script anymore
  if (pCMod == 0) {
    PyErr_Print();
    EXCEPTION_RAISE("ConfigureError",
                    "Did not import root configuration module " + root_module);
  }
 
  PyObject* pProcessClass = PyObject_GetAttrString(pCMod, root_class.c_str());
  Py_DECREF(pCMod);  // don't need the config module anymore
  if (pProcessClass == 0) {
    PyErr_Print();
    EXCEPTION_RAISE("ConfigureError",
                    "Did not import root configuration class " + root_class);
  }
 
  PyObject* pProcess =
      PyObject_GetAttrString(pProcessClass, root_object.c_str());
  Py_DECREF(pProcessClass);  // don't need the Process class anymore
  if (pProcess == 0) {
    // wasn't able to get lastProcess class member
    PyErr_Print();
    EXCEPTION_RAISE(
        "ConfigureError",
        "Process object not defined. This object is required to run.");
  } else if (pProcess == Py_None) {
    // lastProcess was left undefined
    EXCEPTION_RAISE("ConfigureError",
                    "Did not create a configuration class instance");
  }
 
  // okay, now we have fully imported the script and gotten the handle
  // to the last Process object defined in the script.
  // We can now look at pProcess and get all of our parameters out of it.
 
  configuration_.setParameters(getMembers(pProcess));
 
  // all done with python nonsense
  // delete one parent python object
  // MEMORY still not sure if this is enough, but not super worried about it
  //  because this only happens once per run
  Py_DECREF(pProcess);
  // close up python interpreter
  if (Py_FinalizeEx() < 0) {
    PyErr_Print();
    EXCEPTION_RAISE("PyError",
                    "I wasn't able to close up the python interpreter!");
  }
}
 
ProcessHandle ConfigurePython::makeProcess() {
  // no python nonsense happens in here,
  // this just takes the parameters determined earlier
  // and puts them into the Process + EventProcessor framework
 
  return std::make_unique<Process>(configuration_);
}
 
}  // namespace framework