From 61d74fdef836753cbf1cf0e9ee7b8e50cb80f41d Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Mon, 7 Nov 2016 18:13:56 +0300 Subject: [PATCH] tools, unix: Replace upip tarball with just source files. To make its inclusion as frozen modules in multiple ports less magic. Ports are just expected to symlink 2 files into their scripts/modules subdirs. Unix port updated to use this and in general follow frozen modules setup tested and tried on baremetal ports, where there's "scripts" predefined dir (overridable with FROZEN_DIR make var), and a user just drops Python files there. --- tools/micropython-upip-1.1.4.tar.gz | Bin 4090 -> 0 bytes tools/upip.py | 288 ++++++++++++++++++++++++++++ tools/upip_utarfile.py | 94 +++++++++ unix/Makefile | 31 +-- 4 files changed, 385 insertions(+), 28 deletions(-) delete mode 100644 tools/micropython-upip-1.1.4.tar.gz create mode 100644 tools/upip.py create mode 100644 tools/upip_utarfile.py diff --git a/tools/micropython-upip-1.1.4.tar.gz b/tools/micropython-upip-1.1.4.tar.gz deleted file mode 100644 index 588fc4d62ca715b56c7e5e7e07dfdc561317d6b4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4090 zcmb2|=3oE;Cg!){xy6soCVfl)$0f7;nU9)~>QB?9-NCPCeN2g&y|Fg={JW(x)4PNe zXBxT|FTaq{$&P_CLvqTZ|j1lYc0%~$RgGI&TYTV-kx-&)8R*Jr+qtZbME)z z1Q8jYX=lH*EMwjm@#S&JJl=Kx{?z{e^XTr=bL-`udloNduIlUKZ~qqD`}pC2|4QX+fV9v$1nN4v~;oe-#^Vg{ChYSZ{YY{ z|Iq1Jq*dDDgcmU_-garHP45@zNLh3i$LdXA7}0yPbk7;#bDs*n3Fuiww`s~Xoi#pa zC(y5bRJq_@y7#^F8_m+byG}g)F=0bT_wOR9GnD4eyYxnl%;3V`1ibc=&JUkw(Um6nXb^jtC^DOu5&h}icRI5UU%SL z%4r*&N1A+RE~Olum(cObV&%R~n-Yw7q;G5SaGlBiv+L}p4)$}QLShqAEYs%e{i`Yb zd7b-t{J#IG<$qSsI3NDs-0s`cAM&!Y@A_Z;xPRC>jKA;O_K*Lod3MyOajAu0GF;ys z`86zUQtPDDuR$kUJ&b1s>wo=nUT9%MWYx9FA`D;I?_>rYy6{=zAD`FfS2}4OCsiJ9 zKJ;1kc;}BTF&l)x>G`OXKN2aQtRj5mq^+9%rYY8&TiP>c`e$rE^5pH@%@uu*A`K^= z{~;s0asAB~7YZ|KjdY*Sn7ximj!SUep2HXA{AQ{yobn)S&*@VpR>Dfc_ijFy439h@ zZ_T63&e>_49%(nRQb=6RGcoM3O6zu=wNXC`3?sHd#y{nXHoVi2xUg+-2YXUBx4RdY@mYFT=`+Y-c$@A+`zRo9u zpHxO0I^Vx0xm)JBsP*qTk7ud;?(+F9x_E8Z)f7p)4;o^Z7O#|BYEfr&rmw$WdfC^h zm&{{luYT%mckbonvO`X+L57hh__LCKZh!o5MOehe!wj#dO`mpd%A$0honeCJCO7!X z{gl=64^I3$`_@IBY_oXtnpd5Xe~qH~zlH&fQ_>*WaI4pYu8JdHcpEvO=cQ6Y`|^-K$%8 zznXi6n#ZNi9NATs#c~0|uHl}5B zgGdN9n@&5mcxg8JmeokC^W_651P=A<|;?#dPyf!y-b&LO9suX%Y`m6Mj%p>;N z;mfa>?u-6DV}Fb1$*Gr1K09fc{_FCNDYpqyRkyVXKg@B;Pqb_K_x|+d`Sx>m{##bt zyX#p~aeXsG`QoVyl%hO7aq54U^ecJYcQDjUWz|{cD4PnYOulyuoI)Kx?tCG%Mye!p zd)|@am-i+{GXL8t@Ni8)-I_el%IVL`iwv%1H7#;B37r@x<{*7yZ(YY{8$WS_<(HQJ zwt2g5Z3_$MF=@UWy|CTtbCT{AX>AUzly3As#kOn?n+Kb@ecasMx%ZcKUVP*u&0l38 zemrbZj@IQRf2L<@Y}vj3$je>qbE~w<)PCQX*|jdp-SGb$W!Za+r#swQ%f0FM`NmHs z+K+`Z9NOYOG%fF+_xs6l)|wf3axDS^m|UvIFmFTVTYqFr<8hqSB*?-I5j|F+d7(Z~B& zN|1JslB3$!FW;J(ozov(vps6XeoaxzvaK#LMw)i;~K7w{K&X zZv593TJd(q-f3@s{bDzJyr|>(M;#rD?xkAt{fR=0&p(}bI!-%#``rus+~)FcIkq;< zzr{bw;_|VBp2bCME47r@#1t5|8ymK;@l06L<5*%_wBLSrBD2Sjf6ty}8F01Pch0}` zeL~BTW#zxrU)nroyQN*Dn~}7~v}cmVQ?-Sfv+ar+3p5Wcn!*r2=|S;R>y{bSIqThK z&*~C7cIB=#-(53{OWo>QDhzB2taWRbN{6IgvOJisy+Gu}hEr3QTV^rwGV4sZy>de6 z`|EjC%{zZUrP)U`Le;i-U9_2@I&;kxvq=~BDV#jy zG3k5u%4ZW)J{GJEO z<{RH9p~j(+dE5PXx9+<>@vY|U)74x4%sF!Tf$s8uPySq*-CwmPFOm1AU3S(lUyXa3 z5qetkZ9=g4NpHM%~#_m^FDTQ+U;#E;K^pM4$oT{mY+?BN^Qn>s6V<;$FI9<#Wv z{Y}j$O;u8C*HKI5uMe~rMi@%k7qFCKU z9p9$zrAi)*b~%&P42$XxFNth2*nal9cU@dcGV49FyAMQ}|4e_QsXe*F*Yb4WniXC5 ziyM^;b-Be-U+UFa#+WCqC_ic&Z#7Nr#Fhz;^Y3KkmK5c;+|I~}KQPbj&wm}uUCs+S z+MIV=g)j6944lkTIj#C|Z`{UZtYT}P-m{TXV0GA@bNJQP3BR29a_2|M3b$C^Ja=m0 z{-b;)Rn_dKW)s@QINnw6y6kxR*`oCmTRV!^KkkUho0nj5QMqG6rP2ezgt!09A1v9} z#Qk9-=ULri=Hrv|jyPSO`R|L8_B}zNi0r`C^PJ9ZikzW!q9JYi+kMk^hlZufRk<#y zUZp?B<8NHB=d_1jmReD#C4|;UIq!b6eAY8U)Cx*S+yV!bp^3AM5v%BX~o=rR9bzG7wFY4LU ztl}AA;<_zOk-@psp31K^2>ZL7=ak6ICkLZnzdn;2AGiK)u{clt;(Y0+tZS!xxwp*} z`aDm4Yfswp*Z-bOv3%t+KX&m!C!?KFv2(BcZQ=>GX}NTQ>B%|u&X=$JaMUtMwO&s`(>6$>A0w!vKFEq*Gs*F?zFecAK0`n_l0@euZayN zr=M+On9{h`zy5ENjEmKpxl21-cI&Px{o)lFH0PVhr`|W!FYBLf4!9yK+%L1sJGp1V z0!RP*(NhX@9h^I3FYgdwk6>j+8&CI??n_-2;2!3pxTRx?+}(I}-sc=n2u z`7H%*W&F!Bq?NDEe|_ww!GzQ$R)1B@PJjI?d$Tn50H5M*o1;?xU4>Ki>TR7};>5UD+;9&*t6tOu3M9>FlD+%H3gWUgb^j4tmV>pVjzl zXw2H1US}PnE*-t4eqF@4?vG}z$L8fF0n4_&7OGca?2Y}n@8L<&KN)#RHP3Gs?BcGi z(|l*POY;84n`;%f?%HD2$J}+Y#pq7TA|Iof0pDi&^!S!&6_x#yIDGEnlmtri={l_cFKYj0%dbCIuub~tp;Ehf zwehu+wRcPDab!gZoNI8j6;u1DWP0OZLUxpcQjqq6CCc;vTK`^ty~gUC4(HO8_Xj@B zIv?`t(#z%@{u6>8B^iBG@qd=#QMKwe%bQs$Eu5m#au?h0r${W~eKzIe?ci%+I#Kbt zM!L5<&&o`?^<-;BNI*}lC%U(4f{Ty{Ge9vt+1(rM>L*3{9SMj!eGZX5o z_`68ffhBfY&CJfL$!F$YS$F&6uVvQhmkVwgcval`B7L@D<}Hp-j8o@ryIFUm)Ul)W z*y&s5_m*BO3H=bzQSh|4@74-e=XqaaU&R=)Yb~#NxAAMNZ_SpF_WxxUY_<2U=Ge=5 zw9@K))J4vgUGBmv&uP9q$ljy3%P;R8F|hUDN!DPfPB-s`0od5|Dk@ zsx0Kfvcm6wF9a>vbLxEoxAhwSi5uqf$#9?Yz4t6n$}qcueO;*Pfwhmet`kcX;nWM< z88YEZMDX-kk3Js$dG;)S`sB^s+n+BNubcfRjEsM)ilmv^lZjk#&8K zorBW] ... | -r +import upip; upip.install(package_or_list, []) + +If is not given, packages will be installed into sys.path[1] +(can be set from MICROPYPATH environment variable, if current system +supports that).""") + print("Current value of sys.path[1]:", sys.path[1]) + print("""\ + +Note: only MicroPython packages (usually, named micropython-*) are supported +for installation, upip does not support arbitrary code in setup.py. +""") + +def main(): + global debug + global install_path + install_path = None + + if len(sys.argv) < 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help": + help() + return + + if sys.argv[1] != "install": + fatal("Only 'install' command supported") + + to_install = [] + + i = 2 + while i < len(sys.argv) and sys.argv[i][0] == "-": + opt = sys.argv[i] + i += 1 + if opt == "-h" or opt == "--help": + help() + return + elif opt == "-p": + install_path = sys.argv[i] + i += 1 + elif opt == "-r": + list_file = sys.argv[i] + i += 1 + with open(list_file) as f: + while True: + l = f.readline() + if not l: + break + to_install.append(l.rstrip()) + elif opt == "--debug": + debug = True + else: + fatal("Unknown/unsupported option: " + opt) + + to_install.extend(sys.argv[i:]) + if not to_install: + help() + return + + install(to_install) + + if not debug: + cleanup() + + +if __name__ == "__main__": + main() diff --git a/tools/upip_utarfile.py b/tools/upip_utarfile.py new file mode 100644 index 000000000..65ce0bdca --- /dev/null +++ b/tools/upip_utarfile.py @@ -0,0 +1,94 @@ +import uctypes + +# http://www.gnu.org/software/tar/manual/html_node/Standard.html +TAR_HEADER = { + "name": (uctypes.ARRAY | 0, uctypes.UINT8 | 100), + "size": (uctypes.ARRAY | 124, uctypes.UINT8 | 12), +} + +DIRTYPE = "dir" +REGTYPE = "file" + +def roundup(val, align): + return (val + align - 1) & ~(align - 1) + +class FileSection: + + def __init__(self, f, content_len, aligned_len): + self.f = f + self.content_len = content_len + self.align = aligned_len - content_len + + def read(self, sz=65536): + if self.content_len == 0: + return b"" + if sz > self.content_len: + sz = self.content_len + data = self.f.read(sz) + sz = len(data) + self.content_len -= sz + return data + + def readinto(self, buf): + if self.content_len == 0: + return 0 + if len(buf) > self.content_len: + buf = memoryview(buf)[:self.content_len] + sz = self.f.readinto(buf) + self.content_len -= sz + return sz + + def skip(self): + sz = self.content_len + self.align + if sz: + buf = bytearray(16) + while sz: + s = min(sz, 16) + self.f.readinto(buf, s) + sz -= s + +class TarInfo: + + def __str__(self): + return "TarInfo(%r, %s, %d)" % (self.name, self.type, self.size) + +class TarFile: + + def __init__(self, name=None, fileobj=None): + if fileobj: + self.f = fileobj + else: + self.f = open(name, "rb") + self.subf = None + + def next(self): + if self.subf: + self.subf.skip() + buf = self.f.read(512) + if not buf: + return None + + h = uctypes.struct(uctypes.addressof(buf), TAR_HEADER, uctypes.LITTLE_ENDIAN) + + # Empty block means end of archive + if h.name[0] == 0: + return None + + d = TarInfo() + d.name = str(h.name, "utf-8").rstrip() + d.size = int(bytes(h.size).rstrip(), 8) + d.type = [REGTYPE, DIRTYPE][d.name[-1] == "/"] + self.subf = d.subf = FileSection(self.f, d.size, roundup(d.size, 512)) + return d + + def __iter__(self): + return self + + def __next__(self): + v = self.next() + if v is None: + raise StopIteration + return v + + def extractfile(self, tarinfo): + return tarinfo.subf diff --git a/unix/Makefile b/unix/Makefile index fd98d2ced..a14ed215c 100644 --- a/unix/Makefile +++ b/unix/Makefile @@ -1,6 +1,8 @@ -include mpconfigport.mk include ../py/mkenv.mk +FROZEN_DIR = scripts + # define main target PROG = micropython @@ -148,17 +150,6 @@ SRC_C = \ fatfs_port.c \ $(SRC_MOD) -# Include builtin package manager in the standard build (and coverage) -ifeq ($(PROG),micropython) -SRC_C += $(BUILD)/_frozen_upip.c -else ifeq ($(PROG),micropython_coverage) -SRC_C += $(BUILD)/_frozen_upip.c -else ifeq ($(PROG), micropython_nanbox) -SRC_C += $(BUILD)/_frozen_upip.c -else ifeq ($(PROG), micropython_freedos) -SRC_C += $(BUILD)/_frozen_upip.c -endif - LIB_SRC_C = $(addprefix lib/,\ $(LIB_SRC_C_EXTRA) \ timeutils/timeutils.c \ @@ -235,7 +226,7 @@ fast: # build a minimal interpreter minimal: $(MAKE) COPT="-Os -DNDEBUG" CFLAGS_EXTRA='-DMP_CONFIGFILE=""' \ - BUILD=build-minimal PROG=micropython_minimal \ + BUILD=build-minimal PROG=micropython_minimal FROZEN_DIR= \ MICROPY_PY_BTREE=0 MICROPY_PY_FFI=0 MICROPY_PY_SOCKET=0 MICROPY_PY_THREAD=0 \ MICROPY_PY_TERMIOS=0 MICROPY_PY_USSL=0 \ MICROPY_USE_READLINE=0 MICROPY_FATFS=0 @@ -272,22 +263,6 @@ coverage_test: coverage gcov -o build-coverage/py ../py/*.c gcov -o build-coverage/extmod ../extmod/*.c -$(BUILD)/_frozen_upip.c: $(BUILD)/frozen_upip/upip.py - $(MAKE_FROZEN) $(dir $^) > $@ - -# Select latest upip version available -UPIP_TARBALL := $(shell ls -1 -v ../tools/micropython-upip-*.tar.gz | tail -n1) - -$(BUILD)/frozen_upip/upip.py: $(UPIP_TARBALL) - $(ECHO) "MISC Preparing upip as frozen module" - $(Q)mkdir -p $(BUILD) - $(Q)rm -rf $(BUILD)/micropython-upip-* - $(Q)tar -C $(BUILD) -xz -f $^ - $(Q)rm -rf $(dir $@) - $(Q)mkdir -p $(dir $@) - $(Q)cp $(BUILD)/micropython-upip-*/upip*.py $(dir $@) - - # Value of configure's --host= option (required for cross-compilation). # Deduce it from CROSS_COMPILE by default, but can be overriden. ifneq ($(CROSS_COMPILE),)