Using imp.load_source to dynamically load python modules AND packages -

i'm trying dynamically load modules , packages arbitrary folder locations in python 2.7. works great bare, single file modules. trying load in package bit harder.

the best figure out load init.py file inside package (folder). example have this:

root:   mod.py   package:     __init__.py     sub.py 

if mod.py contains:

from package import sub 

using current loading code (below), fail stating there no package named "sub", unless add following package/__init__.py

import sub 

i have imagine because when import package scan other sub files in it. need manually, or there method similar imp.load_source handle package folders?

loading code:

import md5 import sys import os.path import imp import traceback import glob  def load_package(path, base):     try:         try:             sys.path.append(path + "/" + base)             init = path + "/" + base + "/__init__.py"             if not os.path.exists(init):                 return none              fin = open(init, 'rb')              return  (base, imp.load_source(base, init, fin))         finally:             try: fin.close()             except: pass     except importerror, x:         traceback.print_exc(file = sys.stderr)         raise     except:         traceback.print_exc(file = sys.stderr)         raise  def load_module(path):     try:         try:             code_dir = os.path.dirname(path)             code_file = os.path.basename(path)             base = code_file.replace(".py", "")              fin = open(path, 'rb')              hash = md5.new(path).hexdigest() + "_" + code_file             return  (base, imp.load_source(base, path, fin))         finally:             try: fin.close()             except: pass     except importerror, x:         traceback.print_exc(file = sys.stderr)         raise     except:         traceback.print_exc(file = sys.stderr)         raise  def load_folder(dir):     sys.path.append(dir)     mods = {}      p in glob.glob(dir + "/*/"):         base = p.replace("\\", "").replace("/", "")         base = base.replace(dir.replace("\\", "").replace("/", ""), "")         package = load_package(dir, base)          if package:             hash, pack = package             mods[hash] = pack      m in glob.glob(dir + "/*.py"):         hash, mod = load_module(m)          mods[hash] = mod      return mods 

the code below functionally equivalent code modulo traceback.print_exc (which should let client handle - if not handled exception end printed anyway):

def _load_package(path, base):     sys.path.append(path + "/" + base)     init = path + "/" + base + "/__init__.py"     if not os.path.exists(init):         return none, none     open(init, 'rb') fin:         return base, imp.load_source(base, init, fin)  def _load_module(path):     code_file = os.path.basename(path)     base = code_file.replace(".py", "")     open(path, 'rb') fin:         return base, imp.load_source(base, path, fin)  def load_folder(dir):     sys.path.append(dir)     mods = {}     p in glob.glob(dir + "/*/"):         base = p.replace("\\", "").replace("/", "")         base = base.replace(dir.replace("\\", "").replace("/", ""), "")         hash, pack = _load_package(dir, base)         if hash: mods[hash] = pack     m in glob.glob(dir + "/*.py"): ##: /*/*.py         hash, mod = _load_module(m)         mods[hash] = mod     return mods  ## added code print('python %s on %s' % (sys.version, sys.platform))  root_ = r'c:\dropbox\eclipse_workspaces\python\sandbox\root'  def depyc(root, _indent=''): # deletes .pyc end being imported     if not _indent: print '\nlisting', root     p in os.listdir(root):         name = _indent + p         abspath = os.path.join(root, p)         if os.path.isdir(abspath):             print name + ':'             depyc(abspath, _indent=_indent + '  ')         else:             name_ = name[-4:]             if name_ == '.pyc':                 os.remove(abspath)                 continue             print name     if not _indent: print  depyc(root_) load_folder(root_) 

prints:

python 2.7.10 (default, may 23 2015, 09:40:32) [msc v.1500 32 bit (intel)] on win32  listing c:\dropbox\eclipse_workspaces\python\sandbox\root mod.py package:   sub.py   __init__.py  c:\dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py imported! c:\dropbox\eclipse_workspaces\python\sandbox\root\mod.py imported! 

mod.py, sub.py , __init__.py contain

print(__file__ + u' imported!') 

now modifying mod.py to:

from package import sub print(__file__ + u' imported!') 

we indeed:

listing....  c:\dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py imported! <### may move around ###> traceback (most recent call last):   file "c:/users/mrd/.pycharm40/config/scratches/load_folder.py", line 57, in <module>     load_folder(root_)   file "c:/users/mrd/.pycharm40/config/scratches/load_folder.py", line 31, in load_folder     hash, mod = _load_module(m)   file "c:/users/mrd/.pycharm40/config/scratches/load_folder.py", line 20, in _load_module     return base, imp.load_source(base, path, fin)   file "c:\dropbox\eclipse_workspaces\python\sandbox\root\mod.py", line 1, in <module>     package import sub importerror: cannot import name sub 

note error "cannot import name sub" , not "there no package named "sub"". why can't ?

modifying __init__.py:

# package/__init__.py     print(__file__ + u' imported!')  print '__name__', '->', __name__ print '__package__', '->', __package__ print '__path__', '->', __path__ 

prints:

listing...  c:\dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py imported! <### not ###> __name__ -> package __package__ -> none __path__ -> traceback (most recent call last):   file "c:/users/mrd/.pycharm40/config/scratches/load_folder.py", line 59, in <module>     load_folder(root_)   file "c:/users/mrd/.pycharm40/config/scratches/load_folder.py", line 30, in load_folder     hash, pack = _load_package(dir, base)   file "c:/users/mrd/.pycharm40/config/scratches/load_folder.py", line 14, in _load_package     init = imp.load_source(base, init, fin)   file "c:\dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py", line 5, in <module>     print '__path__', '->', __path__ nameerror: name '__path__' not defined 

while directly importing print:

>>> sys.path.extend([r'c:\dropbox\eclipse_workspaces\python\sandbox\root']) >>> import package c:\dropbox\eclipse_workspaces\python\sandbox\root\package\__init__.py imported! __name__ -> package __package__ -> none __path__ -> ['c:\\dropbox\\eclipse_workspaces\\python\\sandbox\\root\\package'] 

so modify _load_package to:

def _load_package(path, base):     pkgdir = os.path.abspath(os.path.join(path, base))     init = os.path.join(pkgdir, "__init__.py")     if not os.path.exists(init):         return none, none     file, pathname, description = imp.find_module(base, [path])     print file, pathname, description # none, pkgdir, ('', '', 5)     pack = sys.modules.get(base, none) # load_module reload - yak!     if pack none:         sys.modules[base] = pack = imp.load_module(base, file, pathname, description)     return base, pack 

solves would:

...     if pack none:         sys.modules[base] = pack = imp.load_module(base, none, '', description)         pack.__path__ = [pkgdir] 

or in original code:

with open(init, 'rb') fin:     source = imp.load_source(base, init, fin)     source.__path__ = path + "/" + base     return base, source 

so what's going on package relies on __path __ attribute function correctly.

---

kept hacking on , came with:

import sys import os.path import imp  def _load_(root, name):     file_object, pathname, description = imp.find_module(name, [root])     pack = sys.modules.get(name, none)     try:         if pack none:             pack = imp.load_module(name, file_object, pathname, description)         else:             print 'in cache', pack     finally:         if file_object not none: file_object.close()     return name, pack  def load_folder(root):     # sys.path.append(root)     mods = {}     paths = [(item, os.path.join(root, item)) item in os.listdir(root)]     packages = filter(lambda path_tuple: os.path.exists(         os.path.join((path_tuple[1]), "__init__.py")), paths)     py_files = filter(lambda path_tuple: path_tuple[0][-3:] == '.py', paths)     del paths     # first import packages in original - modules may import them     path, _abspath in packages:         print 'importing', _abspath         key, mod = _load_(root, name=path) # use pyc if available!         mods[key] = mod     # modules     path, _abspath in py_files:         print 'importing', _abspath         key, mod = _load_(root, name=path[:-3])         mods[key] = mod     return mods 

i merged package , modules loading code dropping imp.load_source (one less tricky function) , relying on imp.load_module instead. not mess sys.path directly , since imp.load_module reload [!] check sys.modules cache. mods dict returned completelly untested - have somehow implement hash (the _abspath should suffice).

run as:

def depyc(root, rmpyc, _indent=''):     if not _indent: print '\nlisting', root     p in os.listdir(root):         name = _indent + p         abspath = os.path.join(root, p)         if os.path.isdir(abspath):             print name + ':'             depyc(abspath, rmpyc, _indent=_indent + '  ')         else:             if rmpyc , name[-4:] == '.pyc':                 os.remove(abspath)                 continue             print name     if not _indent: print  ## run ## print('python %s on %s' % (sys.version, sys.platform)) root_ = os.path.join(os.getcwdu(), u'root') depyc(root_, false) # false end importing pyc files ! load_folder(root_) 

to test various scenarios -

the code example root/ dir here