From: SnickerSun Date: Thu, 1 Aug 2024 03:31:10 +0000 (+0800) Subject: ota脚本修改,资源文件制作采用文件file.bin对比,而非 git对比 X-Git-Tag: release_V1.2.70_20241020_135402~713^2~6^2~2 X-Git-Url: http://git.snicker.com.cn/css/static/gitweb.css?a=commitdiff_plain;h=9ea3685751d7ff6e81838c05585fc68c1cfb0176;p=mirrors%2FFD230.git ota脚本修改,资源文件制作采用文件file.bin对比,而非 git对比 --- diff --git a/mkota.sh b/mkota.sh index 728dd0e80..4d789f5dd 100755 --- a/mkota.sh +++ b/mkota.sh @@ -49,8 +49,9 @@ NEW_COMMIT_VERSION=`cat code/sdk/output/brandy/TestOTA/newsoftware/readme.txt | TMPDIR=../PATCHES/${ROOTDIR}_FSOTA_F rm -rf $TMPDIR -echo ./gen_fs_otapkg.sh $OLD_COMMIT_VERSION $NEW_COMMIT_VERSION F -./gen_fs_otapkg.sh $OLD_COMMIT_VERSION $NEW_COMMIT_VERSION F +echo ./tools/scripts/gen_fs_otapkg.sh $OLD_COMMIT_VERSION $NEW_COMMIT_VERSION F +#./tools/scripts/gen_fs_otapkg_git.sh $OLD_COMMIT_VERSION $NEW_COMMIT_VERSION F +./tools/scripts/gen_fs_otapkg.sh $OLD_COMMIT_VERSION $NEW_COMMIT_VERSION F if [ -d "$TMPDIR" ] ; then RES_DIR=$TMPDIR fi diff --git a/tools/scripts/compare_folder.py b/tools/scripts/compare_folder.py new file mode 100644 index 000000000..bbb090e76 --- /dev/null +++ b/tools/scripts/compare_folder.py @@ -0,0 +1,143 @@ +# -*-coding:utf-8-*- + +#=============================================================================== +# 目录对比工具(包含子目录 ),并列出 +# 1、A比B多了哪些文件 +# 2、B比A多了哪些文件 +# 3、二者相同的文件: md5比较 +#=============================================================================== + +import os +import sys +import time +import difflib +import hashlib +import struct +import shutil +###### fs +#new +onlyInA = [] +#diff +diffAB = [] + +############ diff/deletedfiles.txt +#old +onlyInB = [] + + +def getFileMd5(filename): + if not os.path.isfile(filename): + print('file not exist: ' + filename) + return + myhash = hashlib.md5() + f = open(filename,'rb') + while True: + b = f.read(8096) + if not b : + break + myhash.update(b) + f.close() + return myhash.hexdigest() + + +def getAllFiles(path): + flist=[] + for root, dirs , fs in os.walk(path): + for f in fs: + f_fullpath = os.path.join(root, f) + f_relativepath = f_fullpath[len(path):] + flist.append(f_relativepath) + return flist + +def dirCompare(apath,bpath): + afiles = getAllFiles(apath) + bfiles = getAllFiles(bpath) + + setA = set(afiles) + setB = set(bfiles) + + commonfiles = setA & setB # 处理共有文件 + for f in sorted(commonfiles): + amd=getFileMd5(apath+'/'+f) + bmd=getFileMd5(bpath+'/'+f) + if amd != bmd: + diffAB.append(f) + + # 处理仅出现在一个目录中的文件 + onlyFiles = setA ^ setB + for of in onlyFiles: + if of in afiles: + onlyInA.append(of) + elif of in bfiles: + onlyInB.append(of) + + # if len(diffAB) > 0: + # print ('-' * 20,"diff files ", '-' * 20) + # for of in sorted(diffAB): + # print (of) + + # if len(onlyInA) > 0: + # print ('-' * 20,"only in ", apath, '-' * 20) + # for of in sorted(onlyInA): + # print (of) + + # if len(onlyInB) > 0: + # print ('-' * 20,"only in ", bpath, '-' * 20) + # for of in sorted(onlyInB): + # print (of) + +def create_dirs(dir_path): + # print(dir_path) + if not os.path.exists(dir_path): + try: + os.makedirs(dir_path) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + +def creat_fs_patch(aPath,bPath,target_path): + srca_path=aPath + target_fs_path=os.path.join(target_path, "fs") + if len(onlyInA) > 0: + create_dirs(target_fs_path) + for of in sorted(onlyInA): + basedir=os.path.dirname(of) + target_dir=target_fs_path+basedir + create_dirs(target_dir) + src_file=aPath+of + cmd = "cp " + src_file +" "+target_dir + os.system(cmd) + # print(cmd) + + if len(diffAB) > 0: + create_dirs(target_fs_path) + for of in sorted(diffAB): + basedir=os.path.dirname(of) + target_dir=target_fs_path+basedir + create_dirs(target_dir) + src_file=aPath+of + cmd = "cp " + src_file +" "+target_dir + os.system(cmd) + # print(cmd) + + if len(onlyInB) >0: + create_dirs(target_fs_path+"/diff") + delfiles_path=target_fs_path+"/diff/deletedfiles.txt" + f = open(delfiles_path,'w') + # f.writelines(onlyInB) + for of in sorted(onlyInB): + target_dir=os.path.dirname(of) + # print(of) + f.write(of+"\n") + f.close() + +if __name__ == '__main__': + aPath = sys.argv[1] + bPath = sys.argv[2] + target_tmp_dir = sys.argv[3] + print("{} {} vs {} to {}".format(os.getcwd(),aPath,bPath,target_tmp_dir)) + dirCompare(aPath, bPath) + creat_fs_patch(aPath,bPath,target_tmp_dir) + # print("\ndone!") + + diff --git a/tools/scripts/gen_fs_otapkg.sh b/tools/scripts/gen_fs_otapkg.sh new file mode 100644 index 000000000..04672eb0b --- /dev/null +++ b/tools/scripts/gen_fs_otapkg.sh @@ -0,0 +1,40 @@ +#!/bin/bash +#set -ex + +DATE=$(date +%Y%m%d.%H%M%S) +if ! [ "$3" = "" ] +then +DATE=$3 +fi + +HOMEDIR=$PWD +TMPDIR=/tmp/${USER}_${DATE} + +NEWVERSION=`dirname $0`/../../code/sdk/output/brandy/TestOTA/newsoftware +OLDVERSION=`dirname $0`/../../code/sdk/output/brandy/TestOTA/oldsoftware + +UNYAFFS_SCRIPT=$HOMEDIR/tools/scripts/unyaffs2tool.py +COMPARE_FOLDER_SCRIPT=$HOMEDIR/tools/scripts/compare_folder.py + + +echo $0 +echo $NEWVERSION +echo $OLDVERSION +echo TMPDIR=$TMPDIR +echo HOMEDIR=$HOMEDIR + +echo get diff $OLDVERSION ... $NEWVERSION to $TMPDIR + +BACKUPDIR_HOME=$HOMEDIR/../PATCHES/$(basename $HOMEDIR)_FSOTA_${DATE} +rm -rf $BACKUPDIR_HOME +mkdir -p $BACKUPDIR_HOME + +python $UNYAFFS_SCRIPT $NEWVERSION/bin/file.bin $BACKUPDIR_HOME/new +python $UNYAFFS_SCRIPT $OLDVERSION/bin/file.bin $BACKUPDIR_HOME/old +python $COMPARE_FOLDER_SCRIPT $BACKUPDIR_HOME/new $BACKUPDIR_HOME/old $BACKUPDIR_HOME + + +echo done + + + diff --git a/gen_fs_otapkg.sh b/tools/scripts/gen_fs_otapkg_git.sh similarity index 100% rename from gen_fs_otapkg.sh rename to tools/scripts/gen_fs_otapkg_git.sh diff --git a/tools/scripts/gitdiff.sh b/tools/scripts/gitdiff.sh new file mode 100644 index 000000000..646f750f1 --- /dev/null +++ b/tools/scripts/gitdiff.sh @@ -0,0 +1,75 @@ +#!/bin/bash +#set -e +version=HEAD +DATE=$(date +%Y%m%d.%H%M%S) +HOMEDIR=$PWD +TMPDIR=/tmp/${USER}_${DATE} + + +if ! [ "$1" = "" ] +then + +version=$1 + +fi +echo $version + +BACKUPDIR_HOME=$HOMEDIR/../PATCHES/$(basename $HOMEDIR)_${DATE} + + +genpatch() { +TARGETDIR=$1 +tmpfile_changelist=/$TMPDIR/changelist.txt +tmpfile_removedfiles=/$TMPDIR/deletedfiles.sh +tmpfile_modified_or_new_files=/$TMPDIR/modified_or_new_files.txt +tmpfile_local_diff_patch=/$TMPDIR/local_diff_patch.txt +tmpfile_git_commit_file=/$TMPDIR/commit.txt +BACKUPDIR=$BACKUPDIR_HOME/ +BACKUPDIR_DIFF=$BACKUPDIR_HOME/diff/ + +rm -rf $TMPDIR +mkdir -p $TMPDIR +cd $TARGETDIR +git config core.filemode false +git status . -s > $tmpfile_changelist +export count=`wc $tmpfile_changelist -l|awk '{print $1}'` +echo $TARGETDIR $count +if [ "$count" == "0" ] ; then + echo $TARGETDIR no change +else + + git log -n 1 --format="%H" >$tmpfile_git_commit_file + + git status . -s |awk ' ($1 != "D" ) {print $2 }' >$tmpfile_modified_or_new_files + git status . -s |awk ' ($1 == "D" ) {print "rm " $2 }' >$tmpfile_removedfiles + + mkdir -p $BACKUPDIR + mkdir -p $BACKUPDIR_DIFF + git diff . > $tmpfile_local_diff_patch + mv $tmpfile_local_diff_patch $BACKUPDIR_DIFF + mv $tmpfile_git_commit_file $BACKUPDIR_DIFF + + if ! [ $(wc -l $tmpfile_modified_or_new_files|awk '{print $1}' ) == "0" ] ; then + mv $tmpfile_modified_or_new_files $BACKUPDIR_DIFF + cat $BACKUPDIR_DIFF/$(basename $tmpfile_modified_or_new_files)|xargs tar czf $BACKUPDIR/changefiles.tar.gz + tar xzf $BACKUPDIR/changefiles.tar.gz -C $BACKUPDIR + rm $BACKUPDIR/changefiles.tar.gz + else + echo $tmpfile_modified_or_new_files $(wc -l $tmpfile_modified_or_new_files |awk '{print $1}') , skip copy. + fi + + if ! [ $(wc -l $tmpfile_removedfiles|awk '{print $1}') == "0" ] ; then + mv $tmpfile_removedfiles $BACKUPDIR_DIFF + else + echo $tmpfile_removedfiles $(wc -l $tmpfile_removedfiles|awk '{print $1}') , skip copy. + fi +fi +cd $HOMEDIR + +} + +genpatch $PWD +echo done + + + diff --git a/tools/scripts/mkota.sh b/tools/scripts/mkota.sh new file mode 100644 index 000000000..4d789f5dd --- /dev/null +++ b/tools/scripts/mkota.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +if [ "$#" == "2" ] || [ "$#" == "3" ] ; then +SOURCE_DIR=$1 +SOURCE_VERSION=`basename $SOURCE_DIR` +TARGET_DIR=$2 +TARGET_VERSION=`basename $TARGET_DIR` +RES_DIR=$3 +ROOTDIR=`basename $PWD` +ADDITIONAL_RES_DIR=$PWD/code/sdk/output/additional_fs +OTA_ALL_FLAG_PATH=$PWD/code/sdk/output/ota_all +RES_ONLY_FLAG_PATH=$PWD/code/sdk/output/res_only +OTA_DIFF_FLAG_PATH=$PWD/code/sdk/output/ota_diff +if [ ! -d "$SOURCE_DIR" ] ; then +echo "$SOURCE_DIR not exist , skip create ota package" +echo "" +exit 0 +fi +if [ ! -d "$TARGET_DIR" ] ; then +echo "$TARGET_DIR not exist , skip create ota package" +echo "" +exit 0 +fi + +rm $OTA_ALL_FLAG_PATH $RES_ONLY_FLAG_PATH $OTA_DIFF_FLAG_PATH -f +if [ "$3" == "ota_all" ]; then +touch $OTA_ALL_FLAG_PATH +OTA_PKG_HEAD=ota_all_ +elif [ "$3" == "ota_diff" ]; then +touch $OTA_DIFF_FLAG_PATH +OTA_PKG_HEAD=ota_diff_ +elif [ "$3" == "res_only" ]; then +touch $RES_ONLY_FLAG_PATH +OTA_PKG_HEAD=res_only_ +else +OTA_PKG_HEAD=ota_ +fi + + +mkdir code/sdk/output/brandy/TestOTA/ -p +rm -rf code/sdk/output/brandy/TestOTA/* + +cp -r $SOURCE_DIR code/sdk/output/brandy/TestOTA/oldsoftware +cp -r $TARGET_DIR code/sdk/output/brandy/TestOTA/newsoftware + + +OLD_COMMIT_VERSION=`cat code/sdk/output/brandy/TestOTA/oldsoftware/readme.txt | awk '$1 == "commit" { print $2}'` +NEW_COMMIT_VERSION=`cat code/sdk/output/brandy/TestOTA/newsoftware/readme.txt | awk '$1 == "commit" { print $2}'` + +TMPDIR=../PATCHES/${ROOTDIR}_FSOTA_F +rm -rf $TMPDIR +echo ./tools/scripts/gen_fs_otapkg.sh $OLD_COMMIT_VERSION $NEW_COMMIT_VERSION F +#./tools/scripts/gen_fs_otapkg_git.sh $OLD_COMMIT_VERSION $NEW_COMMIT_VERSION F +./tools/scripts/gen_fs_otapkg.sh $OLD_COMMIT_VERSION $NEW_COMMIT_VERSION F +if [ -d "$TMPDIR" ] ; then +RES_DIR=$TMPDIR +fi +# echo resdir = $RES_DIR = $TMPDIR +if [ -d "$RES_DIR" ] ; then +cp -r $RES_DIR/fs code/sdk/output/brandy/TestOTA/res_file +fi + +if [ -e "$RES_DIR/diff/deletedfiles.txt" ]; then +echo cp -f $RES_DIR/diff/deletedfiles.txt code/sdk/output/brandy/TestOTA/delfiles +cp -f $RES_DIR/diff/deletedfiles.txt code/sdk/output/brandy/TestOTA/delfiles +fi + +if [ -d "$ADDITIONAL_RES_DIR/fs" ]; then +echo cp -rf $ADDITIONAL_RES_DIR/fs/ code/sdk/output/brandy/TestOTA/res_file/ +cp -rf $ADDITIONAL_RES_DIR/fs/ code/sdk/output/brandy/TestOTA/res_file/ +else +echo $ADDITIONAL_RES_DIR/fs not exist, you can put additional resource file to there. +fi + +if [ -e "$ADDITIONAL_RES_DIR/delfiles" ]; then +cat $ADDITIONAL_RES_DIR/delfiles >> code/sdk/output/brandy/TestOTA/delfiles +else +echo $ADDITIONAL_RES_DIR/delfiles not exist, you can put filelist to this file , for adding them to del list in fota file. +fi + +cd $PWD/code/sdk/ +./build/config/target_config/brandy/build_brandy_update.py +cd - + +mkdir release/ota -p + +target_fwpkg_filepath=release/ota/${OTA_PKG_HEAD}${SOURCE_VERSION}_2_${TARGET_VERSION}.fwpkg +cp code/sdk/output/brandy/TestOTA/upgrade/update.fwpkg $target_fwpkg_filepath +fwpkg_filesize=`ls $target_fwpkg_filepath -lh|awk '{print $5}'` + +echo create ota file $target_fwpkg_filepath $fwpkg_filesize done +else + +echo "" +echo "usage: $0 source_release target_release [ota_all]" +echo "" + +fi + diff --git a/tools/scripts/unyaffs2tool.py b/tools/scripts/unyaffs2tool.py new file mode 100644 index 000000000..d05e3e3db --- /dev/null +++ b/tools/scripts/unyaffs2tool.py @@ -0,0 +1,185 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +import sys +import time +import struct +import shutil + +""" +介绍: + 本脚本主要实现调用mkyaffs2image工具生成文件镜像 + 请注意使用本脚本时支持的python版本,本脚本只支持python3版本。 + 请根据实际情况配置对应信息,主要是nandflash的信息,以及分区信息 + +""" + +#tools/scripts/unyaffs2tool.py +cur_dir = os.path.dirname(os.path.abspath(__file__)) +#tools/unyaffs_0.9.7/unyaffs +image_tool_path = os.path.realpath(os.path.join(cur_dir, '..','unyaffs_0.9.7')) + +# 配置生成文件镜像工具路径 +unyaffstool = os.path.join(image_tool_path, 'unyaffs') + +# 配置nandflash信息 +cfg_page_size = 2048 +cfg_obb_size = 64 + + +#code/sdk/tools/bin/platform_tool/mkyaffs2image/mkyaffs2image code/sdk/build/config/target_config/brandy/mk_fs_image/fd230/fs/system code/sdk/build/config/target_config/brandy/mk_fs_image/fd230/file_tmp.bin 1 2 0xcc0000 +#code/sdk/tools/bin/platform_tool/mkyaffs2image/mkyaffs2image code/sdk/build/config/target_config/brandy/mk_fs_image/fs/update code/sdk/build/config/target_config/brandy/mk_fs_image/file_tmp.bin 1 2 0x1980000 +#code/sdk/tools/bin/platform_tool/mkyaffs2image/mkyaffs2image code/sdk/build/config/target_config/brandy/mk_fs_image/fs/music code/sdk/build/config/target_config/brandy/mk_fs_image/file_tmp.bin 1 2 0x2640000 +#code/sdk/tools/bin/platform_tool/mkyaffs2image/mkyaffs2image code/sdk/build/config/target_config/brandy/mk_fs_image/fs/user code/sdk/build/config/target_config/brandy/mk_fs_image/file_tmp.bin 1 2 +# pagesize 2048 oobsize 64 +# 配置分区信息,请根据实际情况进行配置,添加格式为[分区路径, 分区大小] +# 分区大小需要根据按照百分系统分配好后的blocks数进行计算,大小=blocks(文件块个数)*pages_per_block(每个文件块包含的页数)*bytes_per_page(每页包含字节数) +# 前面按照比例分好分区,最后分区置成null,即代表剩下所有blocks +# drivers/drivers/driver/nand_flash/nand/nandflash_config.c +# #ifndef MEMORY_MINI +# #define YF_NAME_STRS "/system/", "/update/", "/music/", "/user/" +# uint32_t yf_volume_percent[YF_VOLUMES_NUM] = { 1, 1, 1, 97 }; +# 0x140000 +# 10 10 10 70 +# 0xcc0000 + + +partition_list = [ + ["system","0x0", "0xcc0000"], + ["update","0xcc0000","0xcc0000"], + ["music", "0x1980000","0xcc0000"], + ["user", "0x2640000","null"] +] + +# 将生成image文件转换大小端,保存成另外文件 +def convert(src_file, dest_file): + fp_src = open(src_file, 'rb') + fp_dest = open(dest_file, 'wb') + src_size = os.path.getsize(src_file) + for i in range(int(src_size / 4)): + data = fp_src.read(4) + rw = struct.pack('BBBB', data[3], data[2], data[1], data[0]) + fp_dest.write(rw) + fp_src.close() + fp_dest.close() + +def splitimg(src_file,dest_file,startaddr,imgsize): + fp_src = open(src_file, 'rb') + fp_dest = open(dest_file, 'wb') + src_size = os.path.getsize(src_file) + print("{} {} {} {}".format(src_file,dest_file,startaddr,imgsize)) + for i in range(int(src_size / 4)): + data = fp_src.read(4) + if i*4 >= startaddr and i*4 < (startaddr+imgsize): + # print(" write {} size {}".format(i*4,readsize)) + rw = struct.pack('BBBB', data[3], data[2], data[1], data[0]) + fp_dest.write(rw) + fp_src.close() + fp_dest.close() + +def unyaffs_img(binname,target_dir): + if not os.path.exists(target_dir): + try: + os.makedirs(target_dir) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + command = "cd " + target_dir +" && " + unyaffstool + " -c 2 -s 64 -b " + binname + " && cd -" + os.system(command) + os.remove(binname) + +def split_img(src_file,target_dir): + if not os.path.exists(target_dir): + try: + os.makedirs(target_dir) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise + fileindex = 0 + fp_src = open(src_file, 'rb') + dest_file = os.path.join(target_dir, "%s_tmp.bin" %partition_list[fileindex][0]) + dest_partition_dir = os.path.join(target_dir, "%s" %partition_list[fileindex][0]) + src_size = os.path.getsize(src_file) + fp_dest = open(dest_file, 'wb') + # YFSS + yaffsheader = struct.pack('BBBB', 0x59,0x46, 0x53, 0x53) + print("read {} ".format(yaffsheader)) + #读取所有数据,一次4字节 + for i in range(int(src_size / 4)): + data = fp_src.read(4) + rw = struct.pack('BBBB', data[3], data[2], data[1], data[0]) + if data[0]==0x53 and data[1]==0x53 and data[2]==0x46 and data[3]==0x59: + #如果是yaffs的header,关闭前一个文件,创建新的对应分区文件, + fp_dest.close() + if i > 0 : + print("write {} size {}".format(dest_file,os.path.getsize(dest_file))) + unyaffs_img(dest_file,dest_partition_dir) + dest_file = os.path.join(target_dir, "%s_tmp.bin" %partition_list[fileindex][0]) + dest_partition_dir = os.path.join(target_dir, "%s" %partition_list[fileindex][0]) + fp_dest = open(dest_file, 'wb') + fileindex+=1 + #读取的4字节写入分区文件 + fp_dest.write(rw) + fp_src.close() + fp_dest.close() + print("write {} size {}".format(dest_file,os.path.getsize(dest_file))) + unyaffs_img(dest_file,dest_partition_dir) + + +def get_page_type(page_size): + if page_size == 2048: + return " 1" + elif page_size == 4096: + return " 2" + else: + return " 0" + +def get_obb_type(obb_size): + if obb_size == 64: + return " 2" + elif obb_size == 128: + return " 4" + else: + return " 0" + +if __name__ == "__main__": + image_file=sys.argv[1] + tmpdir=sys.argv[2] + # image_file = os.path.join(cur_dir, "file.bin") + + if sys.platform == 'linux2' or sys.platform == 'linux': + partition_total = 0 + # page_type = get_page_type(cfg_page_size) + # obb_type = get_obb_type(cfg_obb_size) + page_type = " 1" + obb_type = " 2" + + imgfilesize = os.path.getsize(image_file) + + split_img(image_file,tmpdir) + + # unyaffstool + + # os.system(command) + # convert(tmp_file, image_file) + #unyaffs V0.9.7 - extract files from a YAFFS2 file system image. + + # Usage: unyaffs [options] [] + + # Options: + # -d detection of flash layout, no extraction + # -b spare contains bad block information + # -c set chunk size in KByte (default: autodetect, max: 16) + # -s set spare size in Byte (default: autodetect, max: 512) + # -t list image contents + # -v verbose output + # -V print version + + # unyaffs -c 2 -s 64 -b -t system_tmp.bin + + # if (len(sys.argv) == 2 and os.path.exists(image_file)): + # out_path = os.path.join(sys.argv[1], "%s.bin" %image_name) + # shutil.copyfile(image_file, out_path) + + print("make yaffs image file over!") diff --git a/tools/unyaffs_0.9.7/unyaffs b/tools/unyaffs_0.9.7/unyaffs new file mode 100644 index 000000000..b51016885 Binary files /dev/null and b/tools/unyaffs_0.9.7/unyaffs differ