Using imp.load_source to dynamically load python modules AND packages -
i'm trying dynamically load modules , packages arbitrary folder locations in python 2.7. works great bare, single file modules. trying load in package bit harder.
the best figure out load init.py file inside package (folder). example have this:
root: mod.py package: __init__.py sub.py if mod.py contains:
from package import sub using current loading code (below), fail stating there no package named "sub", unless add following package/__init__.py
import sub i have imagine because when import package scan other sub files in it. need manually, or there method similar imp.load_source handle package folders?
loading code:
import md5 import sys import os.path import imp import traceback import glob def load_package(path, base): try: try: sys.path.append(path + "/" + base) init = path + "/" + base + "/__init__.py" if not os.path.exists(init): return none fin = open(init, 'rb') return (base, imp.load_source(base, init, fin)) finally: try: fin.close() except: pass except importerror, x: traceback.print_exc(file = sys.stderr) raise except: traceback.print_exc(file = sys.stderr) raise def load_module(path): try: try: code_dir = os.path.dirname(path) code_file = os.path.basename(path) base = code_file.replace(".py", "") fin = open(path, 'rb') hash = md5.new(path).hexdigest() + "_" + code_file return (base, imp.load_source(base, path, fin)) finally: try: fin.close() except: pass except importerror, x: traceback.print_exc(file = sys.stderr) raise except: traceback.print_exc(file = sys.stderr) raise def load_folder(dir): sys.path.append(dir) mods = {} p in glob.glob(dir + "/*/"): base = p.replace("\\", "").replace("/", "") base = base.replace(dir.replace("\\", "").replace("/", ""), "") package = load_package(dir, base) if package: hash, pack = package mods[hash] = pack m in glob.glob(dir + "/*.py"): hash, mod = load_module(m) mods[hash] = mod return mods
the code below functionally equivalent code modulo traceback.print_exc (which should let client handle - if not handled exception end printed anyway):
def _load_package(path, base): sys.path.append(path + "/" + base) init = path + "/" + base + "/__init__.py" if not os.path.exists(init): return none, none open(init, 'rb') fin: return base, imp.load_source(base, init, fin) def _load_module(path): code_file = os.path.basename(path) base = code_file.replace(".py", "") open(path, 'rb') fin: return base, imp.load_source(base, path, fin) def load_folder(dir): sys.path.append(dir) mods = {} p in glob.glob(dir + "/*/"): base = p.replace("\\", "").replace("/", "") base = base.replace(dir.replace("\\", "").replace("/", ""), "") hash, pack = _load_package(dir, base) if hash: mods[hash] = pack m in glob.glob(dir + "/*.py"): ##: /*/*.py hash, mod = _load_module(m) mods[hash] = mod return mods ## added code print('python %s on %s' % (sys.version, sys.platform)) root_ = r'c:\dropbox\eclipse_workspaces\python\sandbox\root' def depyc(root, _indent=''): # deletes .pyc end being imported if not _indent: print '\nlisting', root p in os.listdir(root): name = _indent + p abspath = os.path.join(root, p) if os.path.isdir(abspath): print name + ':' depyc(abspath, _indent=_indent + ' ') else: name_ = name[-4:] if name_ == '.pyc': os.remove(abspath) continue print name if not _indent: print depyc(root_) load_folder(root_) prints:
python 2.7.10 (default, may 23 2015, 09:40:32) [msc v.1500 32 bit (intel)] on win32 listing c:\dropbox\eclipse_workspaces\python\sandbox\root mod.py package: sub.py __init__.py c:\dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py imported! c:\dropbox\eclipse_workspaces\python\sandbox\root\mod.py imported! mod.py, sub.py , __init__.py contain
print(__file__ + u' imported!') now modifying mod.py to:
from package import sub print(__file__ + u' imported!') we indeed:
listing.... c:\dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py imported! <### may move around ###> traceback (most recent call last): file "c:/users/mrd/.pycharm40/config/scratches/load_folder.py", line 57, in <module> load_folder(root_) file "c:/users/mrd/.pycharm40/config/scratches/load_folder.py", line 31, in load_folder hash, mod = _load_module(m) file "c:/users/mrd/.pycharm40/config/scratches/load_folder.py", line 20, in _load_module return base, imp.load_source(base, path, fin) file "c:\dropbox\eclipse_workspaces\python\sandbox\root\mod.py", line 1, in <module> package import sub importerror: cannot import name sub note error "cannot import name sub" , not "there no package named "sub"". why can't ?
modifying __init__.py:
# package/__init__.py print(__file__ + u' imported!') print '__name__', '->', __name__ print '__package__', '->', __package__ print '__path__', '->', __path__ prints:
listing... c:\dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py imported! <### not ###> __name__ -> package __package__ -> none __path__ -> traceback (most recent call last): file "c:/users/mrd/.pycharm40/config/scratches/load_folder.py", line 59, in <module> load_folder(root_) file "c:/users/mrd/.pycharm40/config/scratches/load_folder.py", line 30, in load_folder hash, pack = _load_package(dir, base) file "c:/users/mrd/.pycharm40/config/scratches/load_folder.py", line 14, in _load_package init = imp.load_source(base, init, fin) file "c:\dropbox\eclipse_workspaces\python\sandbox\root/package/__init__.py", line 5, in <module> print '__path__', '->', __path__ nameerror: name '__path__' not defined while directly importing print:
>>> sys.path.extend([r'c:\dropbox\eclipse_workspaces\python\sandbox\root']) >>> import package c:\dropbox\eclipse_workspaces\python\sandbox\root\package\__init__.py imported! __name__ -> package __package__ -> none __path__ -> ['c:\\dropbox\\eclipse_workspaces\\python\\sandbox\\root\\package'] so modify _load_package to:
def _load_package(path, base): pkgdir = os.path.abspath(os.path.join(path, base)) init = os.path.join(pkgdir, "__init__.py") if not os.path.exists(init): return none, none file, pathname, description = imp.find_module(base, [path]) print file, pathname, description # none, pkgdir, ('', '', 5) pack = sys.modules.get(base, none) # load_module reload - yak! if pack none: sys.modules[base] = pack = imp.load_module(base, file, pathname, description) return base, pack solves would:
... if pack none: sys.modules[base] = pack = imp.load_module(base, none, '', description) pack.__path__ = [pkgdir] or in original code:
with open(init, 'rb') fin: source = imp.load_source(base, init, fin) source.__path__ = path + "/" + base return base, source so what's going on package relies on __path __ attribute function correctly.
kept hacking on , came with:
import sys import os.path import imp def _load_(root, name): file_object, pathname, description = imp.find_module(name, [root]) pack = sys.modules.get(name, none) try: if pack none: pack = imp.load_module(name, file_object, pathname, description) else: print 'in cache', pack finally: if file_object not none: file_object.close() return name, pack def load_folder(root): # sys.path.append(root) mods = {} paths = [(item, os.path.join(root, item)) item in os.listdir(root)] packages = filter(lambda path_tuple: os.path.exists( os.path.join((path_tuple[1]), "__init__.py")), paths) py_files = filter(lambda path_tuple: path_tuple[0][-3:] == '.py', paths) del paths # first import packages in original - modules may import them path, _abspath in packages: print 'importing', _abspath key, mod = _load_(root, name=path) # use pyc if available! mods[key] = mod # modules path, _abspath in py_files: print 'importing', _abspath key, mod = _load_(root, name=path[:-3]) mods[key] = mod return mods i merged package , modules loading code dropping imp.load_source (one less tricky function) , relying on imp.load_module instead. not mess sys.path directly , since imp.load_module reload [!] check sys.modules cache. mods dict returned completelly untested - have somehow implement hash (the _abspath should suffice).
run as:
def depyc(root, rmpyc, _indent=''): if not _indent: print '\nlisting', root p in os.listdir(root): name = _indent + p abspath = os.path.join(root, p) if os.path.isdir(abspath): print name + ':' depyc(abspath, rmpyc, _indent=_indent + ' ') else: if rmpyc , name[-4:] == '.pyc': os.remove(abspath) continue print name if not _indent: print ## run ## print('python %s on %s' % (sys.version, sys.platform)) root_ = os.path.join(os.getcwdu(), u'root') depyc(root_, false) # false end importing pyc files ! load_folder(root_) to test various scenarios -
the code example root/ dir here
Comments
Post a Comment