diff --git a/.gitignore b/.gitignore index e9d222463..f1d4ce50b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ docker-compose.yml docker-compose.override.yml compose-override.yml build-installer.sh +rsync-to-dev.sh docker-compose-base.yml build-options.json iotstack_build_*.zip diff --git a/.templates/adminer/build.py b/.templates/adminer/build.py index 5a5fb79ad..658897c66 100755 --- a/.templates/adminer/build.py +++ b/.templates/adminer/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -22,7 +22,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -38,43 +37,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -165,13 +127,18 @@ def createMenu(): adminerBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -254,6 +221,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, adminerBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -299,17 +267,50 @@ def menuEntryPoint(): #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'adminer': - main() -else: - print("Error. '{}' Tried to run 'adminer' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/deconz/build.py b/.templates/deconz/build.py index 57e3a53d0..a5c08a7a6 100755 --- a/.templates/deconz/build.py +++ b/.templates/deconz/build.py @@ -1,38 +1,34 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly def main(): import os - import time import ruamel.yaml import signal import sys from blessed import Terminal from deps.chars import specialChars, commonTopBorder, commonBottomBorder, commonEmptyLine - from deps.consts import servicesDirectory, templatesDirectory, buildSettingsFileName, buildCache, servicesFileName - from deps.common_functions import getExternalPorts, getInternalPorts, checkPortConflicts, enterPortNumberWithWhiptail, generateRandomString + from deps.consts import servicesDirectory, buildSettingsFileName + from deps.common_functions import getExternalPorts, getInternalPorts, checkPortConflicts, enterPortNumberWithWhiptail yaml = ruamel.yaml.YAML() yaml.preserve_quotes = True global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring global hideHelpText # Showing and hiding the help controls text global serviceService - global serviceTemplate global hasRebuiltHardwareSelection serviceService = servicesDirectory + currentServiceName - serviceTemplate = templatesDirectory + currentServiceName buildSettings = serviceService + buildSettingsFileName hasRebuiltHardwareSelection = False @@ -45,43 +41,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -95,101 +54,17 @@ def postBuild(): # This function is optional, and will run just before the build docker-compose.yml code. def preBuild(): global dockerComposeServicesYaml - global currentServiceName - with open("{serviceDir}{buildSettings}".format(serviceDir=serviceService, buildSettings=buildSettingsFileName)) as objHardwareListFile: - deconzYamlBuildOptions = yaml.load(objHardwareListFile) - # Password randomisation - # Multi-service: - with open((r'%s/' % serviceTemplate) + servicesFileName) as objServiceFile: - serviceYamlTemplate = yaml.load(objServiceFile) - - oldBuildCache = {} - try: - with open(r'%s' % buildCache) as objBuildCache: - oldBuildCache = yaml.load(objBuildCache) - except: - pass - - buildCacheServices = {} - if "services" in oldBuildCache: - buildCacheServices = oldBuildCache["services"] - - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - - if os.path.exists(buildSettings): - # Password randomisation - if "databasePasswordOption" in deconzYamlBuildOptions: - if ( - deconzYamlBuildOptions["databasePasswordOption"] == "Randomise database password for this build" - or deconzYamlBuildOptions["databasePasswordOption"] == "Randomise database password every build" - or deconzYamlBuildOptions["databasePasswordOption"] == "Use default password for this build" - ): - if deconzYamlBuildOptions["databasePasswordOption"] == "Use default password for this build": - newPassword = "IOtSt4ckDec0nZ" - else: - newPassword = generateRandomString() - for (index, serviceName) in enumerate(serviceYamlTemplate): - dockerComposeServicesYaml[serviceName] = serviceYamlTemplate[serviceName] - if "environment" in serviceYamlTemplate[serviceName]: - for (envIndex, envName) in enumerate(serviceYamlTemplate[serviceName]["environment"]): - envName = envName.replace("%randomPassword%", newPassword) - dockerComposeServicesYaml[serviceName]["environment"][envIndex] = envName - - # Ensure you update the "Do nothing" and other 2 strings used for password settings in 'passwords.py' - if (deconzYamlBuildOptions["databasePasswordOption"] == "Randomise database password for this build"): - deconzYamlBuildOptions["databasePasswordOption"] = "Do nothing" - with open(buildSettings, 'w') as outputFile: - yaml.dump(deconzYamlBuildOptions, outputFile) - else: # Do nothing - don't change password - for (index, serviceName) in enumerate(buildCacheServices): - if serviceName in buildCacheServices: # Load service from cache if exists (to maintain password) - dockerComposeServicesYaml[serviceName] = buildCacheServices[serviceName] - else: - dockerComposeServicesYaml[serviceName] = serviceYamlTemplate[serviceName] - else: - print("Deconz Warning: Build settings file not found, using default password") - time.sleep(1) - newPassword = "IOtSt4ckDec0nZ" - for (index, serviceName) in enumerate(serviceYamlTemplate): - dockerComposeServicesYaml[serviceName] = serviceYamlTemplate[serviceName] - if "environment" in serviceYamlTemplate[serviceName]: - for (envIndex, envName) in enumerate(serviceYamlTemplate[serviceName]["environment"]): - envName = envName.replace("%randomPassword%", newPassword) - dockerComposeServicesYaml[serviceName]["environment"][envIndex] = envName - - deconzYamlBuildOptions["databasePasswordOption"] = "Do nothing" - with open(buildSettings, 'w') as outputFile: - yaml.dump(deconzYamlBuildOptions, outputFile) - - else: - print("Deconz Warning: Build settings file not found, using default password") - time.sleep(1) - newPassword = "IOtSt4ckDec0nZ" - for (index, serviceName) in enumerate(serviceYamlTemplate): - dockerComposeServicesYaml[serviceName] = serviceYamlTemplate[serviceName] - if "environment" in serviceYamlTemplate[serviceName]: - for (envIndex, envName) in enumerate(serviceYamlTemplate[serviceName]["environment"]): - envName = envName.replace("%randomPassword%", newPassword) - dockerComposeServicesYaml[serviceName]["environment"][envIndex] = envName - deconzYamlBuildOptions = { - "version": "1", - "application": "IOTstack", - "service": "Deconz", - "comment": "Deconz Build Options" - } - - deconzYamlBuildOptions["databasePasswordOption"] = "Do nothing" - with open(buildSettings, 'w') as outputFile: - yaml.dump(deconzYamlBuildOptions, outputFile) + if not os.path.exists(buildSettings): + print("DeConz hardware is not configured. Select hardware from Options before building.") + return False try: - if currentServiceName in dockerComposeServicesYaml: - dockerComposeServicesYaml[currentServiceName]["devices"] = deconzYamlBuildOptions["hardware"] - except Exception as err: - print("Error setting deconz hardware: ", err) + with open(buildSettings) as hardwareSettingsFile: + hardwareSettings = yaml.load(hardwareSettingsFile) + dockerComposeServicesYaml[currentServiceName]["devices"] = hardwareSettings["hardware"] + except (OSError, KeyError, TypeError) as err: + print("Error setting DeConz hardware: %s" % err) return False - return True # ##################################### @@ -279,23 +154,6 @@ def enterPortNumberExec(): createMenu() needsRender = 1 - def setPasswordOptions(): - global needsRender - global hasRebuiltAddons - passwordOptionsMenuFilePath = "./.templates/{currentService}/passwords.py".format(currentService=currentServiceName) - with open(passwordOptionsMenuFilePath, "rb") as pythonDynamicImportFile: - code = compile(pythonDynamicImportFile.read(), passwordOptionsMenuFilePath, "exec") - execGlobals = { - "currentServiceName": currentServiceName, - "renderMode": renderMode - } - execLocals = {} - screenActive = False - exec(code, execGlobals, execLocals) - signal.signal(signal.SIGWINCH, onResize) - screenActive = True - needsRender = 1 - def onResize(sig, action): global deconzBuildOptions global currentMenuItemIndex @@ -321,21 +179,22 @@ def createMenu(): deconzBuildOptions.insert(0, ["Change selected hardware", selectDeconzHardware]) else: deconzBuildOptions.insert(0, ["Select hardware", selectDeconzHardware]) - deconzBuildOptions.append([ - "DeConz Password Options", - setPasswordOptions - ]) deconzBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -418,6 +277,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, deconzBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -464,17 +324,50 @@ def menuEntryPoint(): #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'deconz': - main() -else: - print("Error. '{}' Tried to run 'deconz' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/deconz/passwords.py b/.templates/deconz/passwords.py deleted file mode 100755 index 4f4b6315c..000000000 --- a/.templates/deconz/passwords.py +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env python3 - -import signal - -def main(): - from blessed import Terminal - from deps.chars import specialChars, commonTopBorder, commonBottomBorder, commonEmptyLine - from deps.consts import servicesDirectory, templatesDirectory, buildSettingsFileName - import time - import subprocess - import ruamel.yaml - import os - - global signal - global currentServiceName - global menuSelectionInProgress - global mainMenuList - global currentMenuItemIndex - global renderMode - global paginationSize - global paginationStartIndex - global hideHelpText - - yaml = ruamel.yaml.YAML() - yaml.preserve_quotes = True - - try: # If not already set, then set it. - hideHelpText = hideHelpText - except: - hideHelpText = False - - term = Terminal() - hotzoneLocation = [((term.height // 16) + 6), 0] - paginationToggle = [10, term.height - 25] - paginationStartIndex = 0 - paginationSize = paginationToggle[0] - - serviceService = servicesDirectory + currentServiceName - serviceTemplate = templatesDirectory + currentServiceName - buildSettings = serviceService + buildSettingsFileName - - def goBack(): - global menuSelectionInProgress - global needsRender - menuSelectionInProgress = False - needsRender = 1 - return True - - mainMenuList = [] - - hotzoneLocation = [((term.height // 16) + 6), 0] - - menuSelectionInProgress = True - currentMenuItemIndex = 0 - menuNavigateDirection = 0 - - # Render Modes: - # 0 = No render needed - # 1 = Full render - # 2 = Hotzone only - needsRender = 1 - - def onResize(sig, action): - global mainMenuList - global currentMenuItemIndex - mainRender(1, mainMenuList, currentMenuItemIndex) - - def generateLineText(text, textLength=None, paddingBefore=0, lineLength=64): - result = "" - for i in range(paddingBefore): - result += " " - - textPrintableCharactersLength = textLength - - if (textPrintableCharactersLength) == None: - textPrintableCharactersLength = len(text) - - result += text - remainingSpace = lineLength - textPrintableCharactersLength - - for i in range(remainingSpace): - result += " " - - return result - - def renderHotZone(term, renderType, menu, selection, hotzoneLocation, paddingBefore = 4): - global paginationSize - selectedTextLength = len("-> ") - - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) - - if paginationStartIndex >= 1: - print(term.center("{b} {uaf} {uaf}{uaf}{uaf} {ual} {b}".format( - b=specialChars[renderMode]["borderVertical"], - uaf=specialChars[renderMode]["upArrowFull"], - ual=specialChars[renderMode]["upArrowLine"] - ))) - else: - print(term.center(commonEmptyLine(renderMode))) - - for (index, menuItem) in enumerate(menu): # Menu loop - if index >= paginationStartIndex and index < paginationStartIndex + paginationSize: - lineText = generateLineText(menuItem[0], paddingBefore=paddingBefore) - - # Menu highlight logic - if index == selection: - formattedLineText = '-> {t.blue_on_green}{title}{t.normal} <-'.format(t=term, title=menuItem[0]) - paddedLineText = generateLineText(formattedLineText, textLength=len(menuItem[0]) + selectedTextLength, paddingBefore=paddingBefore - selectedTextLength) - toPrint = paddedLineText - else: - toPrint = '{title}{t.normal}'.format(t=term, title=lineText) - # ##### - - # Menu check render logic - if menuItem[1]["checked"]: - toPrint = " (X) " + toPrint - else: - toPrint = " ( ) " + toPrint - - toPrint = "{bv} {toPrint} {bv}".format(bv=specialChars[renderMode]["borderVertical"], toPrint=toPrint) # Generate border - toPrint = term.center(toPrint) # Center Text (All lines should have the same amount of printable characters) - # ##### - print(toPrint) - - if paginationStartIndex + paginationSize < len(menu): - print(term.center("{b} {daf} {daf}{daf}{daf} {dal} {b}".format( - b=specialChars[renderMode]["borderVertical"], - daf=specialChars[renderMode]["downArrowFull"], - dal=specialChars[renderMode]["downArrowLine"] - ))) - else: - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - - - def mainRender(needsRender, menu, selection): - global paginationStartIndex - global paginationSize - term = Terminal() - - if selection >= paginationStartIndex + paginationSize: - paginationStartIndex = selection - (paginationSize - 1) + 1 - needsRender = 1 - - if selection <= paginationStartIndex - 1: - paginationStartIndex = selection - needsRender = 1 - - if needsRender == 1: - print(term.clear()) - print(term.move_y(term.height // 16)) - print(term.black_on_cornsilk4(term.center('IOTstack DeConz Password Options'))) - print("") - print(term.center(commonTopBorder(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Select Password Option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - - if needsRender >= 1: - renderHotZone(term, needsRender, menu, selection, hotzoneLocation) - - if needsRender == 1: - print(term.center(commonEmptyLine(renderMode))) - if not hideHelpText: - if term.height < 32: - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Not enough vertical room to render controls help text {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - else: - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Controls: {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Space] to select option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Up] and [Down] to move selection cursor {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [H] Show/hide this text {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Enter] to build and save option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Escape] to cancel changes {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonBottomBorder(renderMode))) - - def runSelection(selection): - import types - if len(mainMenuList[selection]) > 1 and isinstance(mainMenuList[selection][1], types.FunctionType): - mainMenuList[selection][1]() - else: - print(term.green_reverse('IOTstack Error: No function assigned to menu item: "{}"'.format(mainMenuList[selection][0]))) - - def isMenuItemSelectable(menu, index): - if len(menu) > index: - if len(menu[index]) > 1: - if "skip" in menu[index][1] and menu[index][1]["skip"] == True: - return False - return True - - def loadOptionsMenu(): - global mainMenuList - mainMenuList.append(["Use default password for this build", { "checked": True }]) - mainMenuList.append(["Randomise database password for this build", { "checked": False }]) - mainMenuList.append(["Randomise database password every build", { "checked": False }]) - mainMenuList.append(["Do nothing", { "checked": False }]) - - def checkMenuItem(selection): - global mainMenuList - for (index, menuItem) in enumerate(mainMenuList): - mainMenuList[index][1]["checked"] = False - - mainMenuList[selection][1]["checked"] = True - - def saveOptions(): - try: - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - - if os.path.exists(buildSettings): - with open(r'%s' % buildSettings) as objBuildSettingsFile: - deconzYamlBuildOptions = yaml.load(objBuildSettingsFile) - else: - deconzYamlBuildOptions = { - "version": "1", - "application": "IOTstack", - "service": "deconz", - "comment": "Build Settings", - } - - deconzYamlBuildOptions["databasePasswordOption"] = "" - - for (index, menuOption) in enumerate(mainMenuList): - if menuOption[1]["checked"]: - deconzYamlBuildOptions["databasePasswordOption"] = menuOption[0] - break - - with open(buildSettings, 'w') as outputFile: - yaml.dump(deconzYamlBuildOptions, outputFile) - - except Exception as err: - print("Error saving DeConz Password options", currentServiceName) - print(err) - return False - global hasRebuiltHardwareSelection - hasRebuiltHardwareSelection = True - return True - - def loadOptions(): - try: - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - - if os.path.exists(buildSettings): - with open(r'%s' % buildSettings) as objBuildSettingsFile: - deconzYamlBuildOptions = yaml.load(objBuildSettingsFile) - - for (index, menuOption) in enumerate(mainMenuList): - if menuOption[0] == deconzYamlBuildOptions["databasePasswordOption"]: - checkMenuItem(index) - break - - except Exception as err: - print("Error loading DeConz Password options", currentServiceName) - print(err) - return False - return True - - - if __name__ == 'builtins': - global signal - term = Terminal() - signal.signal(signal.SIGWINCH, onResize) - loadOptionsMenu() - loadOptions() - with term.fullscreen(): - menuNavigateDirection = 0 - mainRender(needsRender, mainMenuList, currentMenuItemIndex) - menuSelectionInProgress = True - with term.cbreak(): - while menuSelectionInProgress: - menuNavigateDirection = 0 - - if not needsRender == 0: # Only rerender when changed to prevent flickering - mainRender(needsRender, mainMenuList, currentMenuItemIndex) - needsRender = 0 - - key = term.inkey(esc_delay=0.05) - if key.is_sequence: - if key.name == 'KEY_TAB': - if paginationSize == paginationToggle[0]: - paginationSize = paginationToggle[1] - else: - paginationSize = paginationToggle[0] - mainRender(1, mainMenuList, currentMenuItemIndex) - if key.name == 'KEY_DOWN': - menuNavigateDirection += 1 - if key.name == 'KEY_UP': - menuNavigateDirection -= 1 - if key.name == 'KEY_ENTER': - if saveOptions(): - return True - else: - print("Something went wrong. Try saving the list again.") - if key.name == 'KEY_ESCAPE': - menuSelectionInProgress = False - return True - elif key: - if key == ' ': # Space pressed - checkMenuItem(currentMenuItemIndex) # Update checked list - needsRender = 2 - elif key == 'h': # H pressed - if hideHelpText: - hideHelpText = False - else: - hideHelpText = True - mainRender(1, mainMenuList, currentMenuItemIndex) - - if menuNavigateDirection != 0: # If a direction was pressed, find next selectable item - currentMenuItemIndex += menuNavigateDirection - currentMenuItemIndex = currentMenuItemIndex % len(mainMenuList) - needsRender = 2 - - while not isMenuItemSelectable(mainMenuList, currentMenuItemIndex): - currentMenuItemIndex += menuNavigateDirection - currentMenuItemIndex = currentMenuItemIndex % len(mainMenuList) - return True - - return True - -originalSignalHandler = signal.getsignal(signal.SIGINT) -main() -signal.signal(signal.SIGWINCH, originalSignalHandler) diff --git a/.templates/deconz/select_hw.py b/.templates/deconz/select_hw.py index 381aa31a5..03834919e 100755 --- a/.templates/deconz/select_hw.py +++ b/.templates/deconz/select_hw.py @@ -90,7 +90,7 @@ def renderHotZone(term, renderType, menu, selection, hotzoneLocation, paddingBef global paginationSize selectedTextLength = len("-> ") - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") if paginationStartIndex >= 1: print(term.center("{b} {uaf} {uaf}{uaf}{uaf} {ual} {b}".format( @@ -273,6 +273,7 @@ def saveAddonList(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, mainMenuList, currentMenuItemIndex) + needsRender = 0 dockerCommandsSelectionInProgress = True with term.cbreak(): while dockerCommandsSelectionInProgress: @@ -325,6 +326,6 @@ def saveAddonList(): return True -originalSignalHandler = signal.getsignal(signal.SIGINT) +originalSignalHandler = signal.getsignal(signal.SIGWINCH) main() signal.signal(signal.SIGWINCH, originalSignalHandler) diff --git a/.templates/deconz/service.yml b/.templates/deconz/service.yml index 2034be992..0ac4cc232 100644 --- a/.templates/deconz/service.yml +++ b/.templates/deconz/service.yml @@ -9,10 +9,10 @@ deconz: volumes: - ./volumes/deconz:/opt/deCONZ devices: - - "${DECONZ_DEVICE_PATH:?eg echo DECONZ_DEVICE_PATH=dev/ttyUSB0 >>~/IOTstack/.env}:/dev/ttyUSB0" + - "/dev/ttyUSB0:/dev/ttyUSB0" environment: - DECONZ_VNC_MODE=1 - - DECONZ_VNC_PASSWORD=${DECONZ_VNC_PASSWORD:-%randomPassword%} + - DECONZ_VNC_PASSWORD=${DECONZ_VNC_PASSWORD:-IOtSt4ckDec0nZ} - DEBUG_INFO=1 - DEBUG_APS=0 - DEBUG_ZCL=0 diff --git a/.templates/diyhue/build.py b/.templates/diyhue/build.py index 7d0ff9b78..52d8c5b46 100755 --- a/.templates/diyhue/build.py +++ b/.templates/diyhue/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -22,7 +22,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -40,43 +39,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -181,13 +143,18 @@ def createMenu(): diyhueBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -261,6 +228,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, diyhueBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -307,17 +275,50 @@ def menuEntryPoint(): #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'diyhue': - main() -else: - print("Error. '{}' Tried to run 'diyhue' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/dozzle/build.py b/.templates/dozzle/build.py index 04208d9ac..dc015f69e 100755 --- a/.templates/dozzle/build.py +++ b/.templates/dozzle/build.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 +OPTIONS_AVAILABLE = False + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -22,7 +23,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -39,43 +39,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -166,13 +129,18 @@ def createMenu(): dozzleBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -246,6 +214,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, dozzleBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -292,17 +261,50 @@ def menuEntryPoint(): #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'dozzle': - main() -else: - print("Error. '{}' Tried to run 'dozzle' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/esphome/build.py b/.templates/esphome/build.py index 641cd87bf..42911e8e9 100755 --- a/.templates/esphome/build.py +++ b/.templates/esphome/build.py @@ -1,175 +1,45 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*- - -issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks -haltOnErrors = True +#!/usr/bin/env python3 import os -import sys - -global templatesDirectory -global currentServiceName # Name of the current service -global generateRandomString +import subprocess from deps.consts import templatesDirectory -from deps.common_functions import generateRandomString - - -# Main wrapper function. Required to make local vars work correctly -def main(): - - global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result - global issues # Returned issues dict - global haltOnErrors # Turn on to allow erroring - - # runtime vars - portConflicts = [] - - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - - # This service will not check anything unless this is set - # This function is optional, and will run each time the menu is rendered - def runChecks(): - checkForIssues() - return [] - - # This function is optional, and will run after the docker-compose.yml file is written to disk. - def postBuild(): - return True - - # This function is optional, and will run just before the build docker-compose.yml code. - def preBuild(): - return True - - # ##################################### - # Supporting functions below - # ##################################### - - def doCustomSetup() : - - import os - import re - import subprocess - from os.path import exists - - def copyUdevRulesFile(templates,rules) : - - # the expected location of the rules file in the template is the absolute path ... - SOURCE_PATH = templates + '/' + currentServiceName + '/' + rules - - # the rules file should be installed at the following absolute path... - TARGET_PATH = '/etc/udev/rules.d' + '/' + rules - - # does the target already exist? - if not exists(TARGET_PATH) : - - # no! does the source path exist? - if exists(SOURCE_PATH) : - - # yes! we should copy the source to the target - subprocess.call(['sudo', 'cp', SOURCE_PATH, TARGET_PATH]) - - # sudo cp sets root ownership but not necessarily correct mode - subprocess.call(['sudo', 'chmod', '644', TARGET_PATH]) - - def setEnvironment (path, key, value) : - - # assume the variable should be written - shouldWrite = True - - # does the target file already exist? - if exists(path) : - - # yes! open the file so we can search it - env_file = open(path, 'r+') - - # prepare to read by lines - env_data = env_file.readlines() - - # we are searching for... - expression = '^' + key + '=' - # search by line - for line in env_data: - if re.search(expression, line) : - shouldWrite = False - break - else : - - # no! create the file - env_file = open(path, 'w') - - # should the variable be written? - if shouldWrite : - print(key + '=' + value, file=env_file) +UDEV_RULES_FILE = "88-tty-iotstack-esphome.rules" +UDEV_RULES_DIRECTORY = "/etc/udev/rules.d" - # done with the environment file - env_file.close() - copyUdevRulesFile( - os.path.realpath(templatesDirectory), - '88-tty-iotstack-' + currentServiceName + '.rules' - ) +def runChecks(context): + return {} - # the environment file is located at ... - DOT_ENV_PATH = os.path.realpath('.') + '/.env' - # check/set environment variables - setEnvironment(DOT_ENV_PATH,'ESPHOME_USERNAME',currentServiceName) - setEnvironment(DOT_ENV_PATH,'ESPHOME_PASSWORD',generateRandomString()) +def preBuild(context): + sourcePath = os.path.realpath(os.path.join( + templatesDirectory, + context.serviceName, + UDEV_RULES_FILE, + )) + targetPath = os.path.join(UDEV_RULES_DIRECTORY, UDEV_RULES_FILE) + if os.path.exists(targetPath): + return True + if not os.path.exists(sourcePath): + print("ESPHome udev rules file is missing: %s" % sourcePath) + return False - def checkForIssues(): - doCustomSetup() # done here because is called least-frequently - return True + try: + result = subprocess.run([ + "sudo", "install", "-m", "0644", sourcePath, targetPath, + ]) + except OSError as err: + print("Unable to install ESPHome udev rules: %s" % err) + return False + if result.returncode != 0: + print("Unable to install ESPHome udev rules (exit status %s)." % result.returncode) + return False + return True - if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass -if currentServiceName == 'esphome': - main() -else: - print("Error. '{}' Tried to run 'plex' config".format(currentServiceName)) +def postBuild(context): + return True diff --git a/.templates/espruinohub/build.py b/.templates/espruinohub/build.py index 84a45e335..8cfafc23d 100755 --- a/.templates/espruinohub/build.py +++ b/.templates/espruinohub/build.py @@ -1,14 +1,13 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -16,43 +15,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -74,17 +36,45 @@ def preBuild(): def checkForIssues(): return True + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass - -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'espruinohub': - main() -else: - print("Error. '{}' Tried to run 'espruinohub' config".format(currentServiceName)) + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + + +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/example_template/README.md b/.templates/example_template/README.md new file mode 100644 index 000000000..ba398ad02 --- /dev/null +++ b/.templates/example_template/README.md @@ -0,0 +1,25 @@ +# Creating an IOTstack service + +1. Copy this entire directory and give the copy your service name. +2. Rename `example_service.yml` to `service.yml`. +3. Change the root key in `service.yml` so it exactly matches the directory name. +4. Edit the Compose settings for the container. +5. Delete `build.py` if the service needs no menu hooks. Otherwise, keep only the hook functions you need. + +The optional hook functions are: + +```python +def runChecks(context): + return {} + +def runOptionsMenu(context): + pass + +def preBuild(context): + pass + +def postBuild(context): + pass +``` + +Run `python3 -m unittest discover -v` before submitting a pull request. diff --git a/.templates/example_template/build.py b/.templates/example_template/build.py new file mode 100755 index 000000000..c047f9967 --- /dev/null +++ b/.templates/example_template/build.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +"""Optional build hooks for a service template. + +Copy this file with the example template, then delete any hooks your service +does not need. IOTstack discovers the functions by name. +""" + + + +def runChecks(context): + """Return build issues as a dictionary. An empty dictionary means pass.""" + from deps.common_functions import checkPortConflicts, getExternalPorts + + issues = {} + currentPorts = getExternalPorts(context.serviceName, context.services) + conflicts = [] + for serviceName in context.services: + if serviceName != context.serviceName: + conflicts.extend(checkPortConflicts(serviceName, currentPorts, context.services)) + + if conflicts: + issues["portConflicts"] = conflicts + return issues + + +def runOptionsMenu(context): + """Example option: change the service's first published port.""" + from deps.common_functions import enterPortNumberWithWhiptail, getExternalPorts, getInternalPorts + + externalPorts = getExternalPorts(context.serviceName, context.services) + internalPorts = getInternalPorts(context.serviceName, context.services) + if not externalPorts or not internalPorts: + return + + newPort = enterPortNumberWithWhiptail( + context.terminal, + context.services, + context.serviceName, + [7, 0], + externalPorts[0], + ) + if newPort != -1: + context.services[context.serviceName]["ports"][0] = "%s:%s" % ( + newPort, + internalPorts[0], + ) + + +def preBuild(context): + """Optional: prepare files or update context.services before output.""" + return None + + +def postBuild(context): + """Optional: perform work after docker-compose.yml has been written.""" + return None diff --git a/.templates/example_template/example_build.py b/.templates/example_template/example_build.py deleted file mode 100755 index f55f8da3e..000000000 --- a/.templates/example_template/example_build.py +++ /dev/null @@ -1,312 +0,0 @@ -#!/usr/bin/env python3 - -# Be warned that globals and variable scopes do not function normally in this Python script. This is because this script is eval'd with exec. - -issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks -haltOnErrors = True - -# Main wrapper function. Required to make local vars work correctly -def main(): - from blessed import Terminal - from deps.chars import specialChars, commonTopBorder, commonBottomBorder, commonEmptyLine # Common functions used when creating menu - import types - import time - - global dockerComposeServicesYaml # The loaded memory YAML of all checked services - global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result - global currentServiceName # Name of the current service - global issues # Returned issues dict - global haltOnErrors # Turn on to allow erroring - global hideHelpText - - try: # If not already set, then set it to prevent errors. - hideHelpText = hideHelpText - except: - hideHelpText = False - - # runtime vars - portConflicts = [] - - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - - # This service will not check anything unless this is set - # This function is optional, and will run each time the menu is rendered - def runChecks(): - checkForIssues() - return [] - - # This is the menu that will run for " >> Options " - def runOptionsMenu(): - menuEntryPoint() - return True - - # This function is optional, and will run after the docker-compose.yml file is written to disk. - def postBuild(): - return True - - # This function is optional, and will run just before the build docker-compose.yml code. - def preBuild(): - return True - - # ##################################### - # Supporting functions below - # ##################################### - - def checkForIssues(): - for (index, serviceName) in enumerate(dockerComposeServicesYaml): - if not currentServiceName == serviceName: # Skip self - currentServicePorts = getExternalPorts(currentServiceName) - portConflicts = checkPortConflicts(serviceName, currentServicePorts) - if (len(portConflicts) > 0): - issues["portConflicts"] = portConflicts - - def getExternalPorts(serviceName): - externalPorts = [] - try: - yamlService = dockerComposeServicesYaml[serviceName] - if "ports" in yamlService: - for (index, port) in enumerate(yamlService["ports"]): - try: - externalAndInternal = port.split(":") - externalPorts.append(externalAndInternal[0]) - except: - pass - except: - pass - return externalPorts - - def checkPortConflicts(serviceName, currentPorts): - portConflicts = [] - if not currentServiceName == serviceName: - yamlService = dockerComposeServicesYaml[serviceName] - servicePorts = getExternalPorts(serviceName) - for (index, servicePort) in enumerate(servicePorts): - for (index, currentPort) in enumerate(currentPorts): - if (servicePort == currentPort): - portConflicts.append([servicePort, serviceName]) - return portConflicts - - - - # ##################################### - # Example menu below - # ##################################### - # You can build your menu system any way you like. This one is provided as an example. - # Checkout Blessed for full functionality, like text entry and so on at: https://blessed.readthedocs.io/en/latest/ - - # The functions the menu executes are below. They must be placed before the menu list 'menuItemsExample' - def menuCmdItem1(): - print("You chose item1!") - return True - - def menuCmdAnotherItem(): - print("This is another menu item") - return True - - def nop(): - return True - - def menuCmdStillAnotherItem(): - print("This is still another menu item") - return True - - def goBack(): - global selectionInProgress - selectionInProgress = False - return True - - # The actual menu - menuItemsExample = [ - ["Item 1", menuCmdItem1], - ["Another item", menuCmdAnotherItem], - ["I'm skipped!", nop, { "skip": True }], - ["Still another item", menuCmdStillAnotherItem], - ["Error item"], - ["Error item"], - ["Some custom thing", nop, { "customProperty": True }], - ["I'm also skipped!", nop, { "skip": True }], - ["Go back", goBack] - ] - - # Vars that the menu uses - global currentMenuItemIndex - global selectionInProgress - global menuNavigateDirection - global needsRender - - selectionInProgress = True - currentMenuItemIndex = 0 - menuNavigateDirection = 0 - needsRender = True - - # This is the main rendering function for the menu - def mainRender(menu, selection): - term = Terminal() - print(term.clear()) - - print(term.clear()) - print(term.move_y(term.height // 16)) - print(term.black_on_cornsilk4(term.center('IOTstack Example Commands'))) - print("") - print(term.center(commonTopBorder(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Select Command to run {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - - print(term.center(commonEmptyLine(renderMode))) - - lineLengthAtTextStart = 71 - - for (index, menuItem) in enumerate(menu): - toPrint = "" - if index == selection: # This checks if the current rendering item is the one that's selected - toPrint += ('{bv} -> {t.blue_on_green} {title} {t.normal} <-'.format(t=term, title=menuItem[0], bv=specialChars[renderMode]["borderVertical"])) - else: - if len(menu[index]) > 2 and "customProperty" in menu[index][2] and menu[index][2]["customProperty"] == True: # A custom property check example - toPrint += ('{bv} {t.black_on_green} {title} {t.normal} '.format(t=term, title=menuItem[0], bv=specialChars[renderMode]["borderVertical"])) - else: - toPrint += ('{bv} {t.normal} {title} '.format(t=term, title=menuItem[0], bv=specialChars[renderMode]["borderVertical"])) - - for i in range(lineLengthAtTextStart - len(menuItem[0])): # Pad the remainder of the line - toPrint += " " - - toPrint += "{bv}".format(bv=specialChars[renderMode]["borderVertical"]) - - toPrint = term.center(toPrint) - - print(toPrint) - - print(term.center(commonEmptyLine(renderMode))) - if not hideHelpText: - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Controls: {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Up] and [Down] to move selection cursor {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [H] Show/hide this text {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Enter] to run command {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Escape] to go back to build stack menu {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonBottomBorder(renderMode))) - - - def runSelection(selection): - term = Terminal() - if len(menuItemsExample[selection]) > 1 and isinstance(menuItemsExample[selection][1], types.FunctionType): - menuItemsExample[selection][1]() - else: - print(term.green_reverse('IOTstack Example Error: No function assigned to menu item: "{}"'.format(menuItemsExample[selection][0]))) - - def isMenuItemSelectable(menu, index): - if len(menu) > index: - if len(menu[index]) > 2: - if "skip" in menu[index][2] and menu[index][2]["skip"] == True: - return False - return True - - def menuEntryPoint(): - # These need to be reglobalised due to eval() - global currentMenuItemIndex - global selectionInProgress - global menuNavigateDirection - global needsRender - global hideHelpText - term = Terminal() - with term.fullscreen(): - menuNavigateDirection = 0 - mainRender(menuItemsExample, currentMenuItemIndex) - selectionInProgress = True - with term.cbreak(): - while selectionInProgress: - menuNavigateDirection = 0 - - if needsRender: # Only rerender when changed to prevent flickering - mainRender(menuItemsExample, currentMenuItemIndex) - needsRender = False - - key = term.inkey(esc_delay=0.05) - if key.is_sequence: - if key.name == 'KEY_TAB': - menuNavigateDirection += 1 - if key.name == 'KEY_DOWN': - menuNavigateDirection += 1 - if key.name == 'KEY_UP': - menuNavigateDirection -= 1 - if key.name == 'KEY_LEFT': - goBack() - if key.name == 'KEY_ENTER': - runSelection(currentMenuItemIndex) - if key.name == 'KEY_ESCAPE': - return True - elif key: - if key == 'h': # H pressed - if hideHelpText: - hideHelpText = False - else: - hideHelpText = True - mainRender(1, menuItemsExample, currentMenuItemIndex) - - if menuNavigateDirection != 0: # If a direction was pressed, find next selectable item - currentMenuItemIndex += menuNavigateDirection - currentMenuItemIndex = currentMenuItemIndex % len(menuItemsExample) - needsRender = True - - while not isMenuItemSelectable(menuItemsExample, currentMenuItemIndex): - currentMenuItemIndex += menuNavigateDirection - currentMenuItemIndex = currentMenuItemIndex % len(menuItemsExample) - return True - - - - - - # Entrypoint for execution - if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass - -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'SERVICENAME': - main() -else: - print("Error. '{}' Tried to run 'SERVICENAME' config".format(currentServiceName)) diff --git a/.templates/gitea/build.py b/.templates/gitea/build.py index e783e8956..b5ac52427 100755 --- a/.templates/gitea/build.py +++ b/.templates/gitea/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -22,7 +22,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -40,43 +39,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -167,13 +129,18 @@ def createMenu(): giteaBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -247,6 +214,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, giteaBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -293,17 +261,50 @@ def menuEntryPoint(): #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'gitea': - main() -else: - print("Error. '{}' Tried to run 'gitea' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/grafana/build.py b/.templates/grafana/build.py index 7da3d8934..1a910d99e 100755 --- a/.templates/grafana/build.py +++ b/.templates/grafana/build.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly def main(): import os + import signal import time from blessed import Terminal from deps.chars import specialChars, commonTopBorder, commonBottomBorder, commonEmptyLine, padText @@ -15,7 +16,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -33,43 +33,6 @@ def main(): documentationHint = 'https://sensorsiot.github.io/IOTstack/Containers/Grafana' - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -156,13 +119,18 @@ def createMenu(): grafanaBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -245,6 +213,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, grafanaBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -290,17 +259,50 @@ def menuEntryPoint(): # End menu section #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'grafana': - main() -else: - print("Error. '{}' Tried to run 'grafana' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/home_assistant/build.py b/.templates/home_assistant/build.py index 2a4e69b77..a3cbbece9 100755 --- a/.templates/home_assistant/build.py +++ b/.templates/home_assistant/build.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 +OPTIONS_AVAILABLE = False + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -22,7 +23,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -41,43 +41,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -168,13 +131,18 @@ def createMenu(): homeAssistantBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -257,6 +225,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, homeAssistantBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -303,17 +272,50 @@ def menuEntryPoint(): #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'home_assistant': - main() -else: - print("Error. '{}' Tried to run 'home_assistant' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/influxdb/build.py b/.templates/influxdb/build.py index c8c5e2b68..12f56a1ce 100755 --- a/.templates/influxdb/build.py +++ b/.templates/influxdb/build.py @@ -1,29 +1,24 @@ #!/usr/bin/env python3 +OPTIONS_AVAILABLE = False + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly def main(): import os - import time - import ruamel.yaml import signal import sys import subprocess from blessed import Terminal from deps.chars import specialChars, commonTopBorder, commonBottomBorder, commonEmptyLine, padText - from deps.consts import servicesDirectory, templatesDirectory, servicesFileName, buildSettingsFileName - from deps.common_functions import getExternalPorts, checkPortConflicts, generateRandomString - - yaml = ruamel.yaml.YAML() - yaml.preserve_quotes = True + from deps.consts import servicesDirectory + from deps.common_functions import getExternalPorts, checkPortConflicts global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -31,8 +26,6 @@ def main(): global serviceService serviceService = servicesDirectory + currentServiceName - serviceTemplate = templatesDirectory + currentServiceName - buildSettings = serviceService + buildSettingsFileName try: # If not already set, then set it. hideHelpText = hideHelpText @@ -44,43 +37,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -93,77 +49,7 @@ def postBuild(): # This function is optional, and will run just before the build docker-compose.yml code. def preBuild(): - # Multi-service: - with open((r'%s/' % serviceTemplate) + servicesFileName) as objServiceFile: - serviceYamlTemplate = yaml.load(objServiceFile) - - oldBuildCache = {} - try: - with open(r'%s' % buildCache) as objBuildCache: - oldBuildCache = yaml.load(objBuildCache) - except: - pass - - buildCacheServices = {} - if "services" in oldBuildCache: - buildCacheServices = oldBuildCache["services"] - - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - - if os.path.exists(buildSettings): - # Password randomisation - with open(r'%s' % buildSettings) as objBuildSettingsFile: - influxDbYamlBuildOptions = yaml.load(objBuildSettingsFile) - if ( - influxDbYamlBuildOptions["databasePasswordOption"] == "Randomise database password for this build" - or influxDbYamlBuildOptions["databasePasswordOption"] == "Randomise database password every build" - or influxDbYamlBuildOptions["databasePasswordOption"] == "Use default password for this build" - ): - if influxDbYamlBuildOptions["databasePasswordOption"] == "Use default password for this build": - randomPassword = "IOtSt4ckInfluX" - else: - randomPassword = generateRandomString() - for (index, serviceName) in enumerate(serviceYamlTemplate): - dockerComposeServicesYaml[serviceName] = serviceYamlTemplate[serviceName] - if "environment" in serviceYamlTemplate[serviceName]: - for (envIndex, envName) in enumerate(serviceYamlTemplate[serviceName]["environment"]): - envName = envName.replace("%randomPassword%", randomPassword) - dockerComposeServicesYaml[serviceName]["environment"][envIndex] = envName - - # Ensure you update the "Do nothing" and other 2 strings used for password settings in 'passwords.py' - if (influxDbYamlBuildOptions["databasePasswordOption"] == "Randomise database password for this build"): - influxDbYamlBuildOptions["databasePasswordOption"] = "Do nothing" - with open(buildSettings, 'w') as outputFile: - yaml.dump(influxDbYamlBuildOptions, outputFile) - else: # Do nothing - don't change password - for (index, serviceName) in enumerate(buildCacheServices): - if serviceName in buildCacheServices: # Load service from cache if exists (to maintain password) - dockerComposeServicesYaml[serviceName] = buildCacheServices[serviceName] - else: - dockerComposeServicesYaml[serviceName] = serviceYamlTemplate[serviceName] - - else: - print("InfluxDB Warning: Build settings file not found, using default password") - time.sleep(1) - randomPassword = "IOtSt4ckInfluX" - for (index, serviceName) in enumerate(serviceYamlTemplate): - dockerComposeServicesYaml[serviceName] = serviceYamlTemplate[serviceName] - if "environment" in serviceYamlTemplate[serviceName]: - for (envIndex, envName) in enumerate(serviceYamlTemplate[serviceName]["environment"]): - envName = envName.replace("%randomPassword%", randomPassword) - dockerComposeServicesYaml[serviceName]["environment"][envIndex] = envName - influxDbYamlBuildOptions = { - "version": "1", - "application": "IOTstack", - "service": "InfluxDB", - "comment": "InfluxDB Build Options" - } - - influxDbYamlBuildOptions["databasePasswordOption"] = "Do nothing" - with open(buildSettings, 'w') as outputFile: - yaml.dump(influxDbYamlBuildOptions, outputFile) - + os.makedirs(serviceService, exist_ok=True) return True # ##################################### @@ -205,23 +91,6 @@ def goBack(): needsRender = 1 return True - def setPasswordOptions(): - global needsRender - global hasRebuiltAddons - passwordOptionsMenuFilePath = "./.templates/{currentService}/passwords.py".format(currentService=currentServiceName) - with open(passwordOptionsMenuFilePath, "rb") as pythonDynamicImportFile: - code = compile(pythonDynamicImportFile.read(), passwordOptionsMenuFilePath, "exec") - execGlobals = { - "currentServiceName": currentServiceName, - "renderMode": renderMode - } - execLocals = {} - screenActive = False - exec(code, execGlobals, execLocals) - signal.signal(signal.SIGWINCH, onResize) - screenActive = True - needsRender = 1 - def onResize(sig, action): global influxDbBuildOptions global currentMenuItemIndex @@ -234,21 +103,22 @@ def createMenu(): global serviceService influxDbBuildOptions = [] - # influxDbBuildOptions.append([ - # "InfluxDB Password Options", - # setPasswordOptions - # ]) influxDbBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -331,6 +201,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, influxDbBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -377,17 +248,50 @@ def menuEntryPoint(): #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass - -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'influxdb': - main() -else: - print("Error. '{}' Tried to run 'influxdb' config".format(currentServiceName)) + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + + +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/influxdb/passwords.py b/.templates/influxdb/passwords.py deleted file mode 100755 index fc27f915e..000000000 --- a/.templates/influxdb/passwords.py +++ /dev/null @@ -1,326 +0,0 @@ -#!/usr/bin/env python3 - -import signal - -def main(): - from blessed import Terminal - from deps.chars import specialChars, commonTopBorder, commonBottomBorder, commonEmptyLine - from deps.consts import servicesDirectory, templatesDirectory, buildSettingsFileName - import time - import subprocess - import ruamel.yaml - import os - - global signal - global currentServiceName - global menuSelectionInProgress - global mainMenuList - global currentMenuItemIndex - global renderMode - global paginationSize - global paginationStartIndex - global hideHelpText - - yaml = ruamel.yaml.YAML() - yaml.preserve_quotes = True - - try: # If not already set, then set it. - hideHelpText = hideHelpText - except: - hideHelpText = False - - term = Terminal() - hotzoneLocation = [((term.height // 16) + 6), 0] - paginationToggle = [10, term.height - 25] - paginationStartIndex = 0 - paginationSize = paginationToggle[0] - - serviceService = servicesDirectory + currentServiceName - serviceTemplate = templatesDirectory + currentServiceName - buildSettings = serviceService + buildSettingsFileName - - def goBack(): - global menuSelectionInProgress - global needsRender - menuSelectionInProgress = False - needsRender = 1 - return True - - mainMenuList = [] - - hotzoneLocation = [((term.height // 16) + 6), 0] - - menuSelectionInProgress = True - currentMenuItemIndex = 0 - menuNavigateDirection = 0 - - # Render Modes: - # 0 = No render needed - # 1 = Full render - # 2 = Hotzone only - needsRender = 1 - - def onResize(sig, action): - global mainMenuList - global currentMenuItemIndex - mainRender(1, mainMenuList, currentMenuItemIndex) - - def generateLineText(text, textLength=None, paddingBefore=0, lineLength=64): - result = "" - for i in range(paddingBefore): - result += " " - - textPrintableCharactersLength = textLength - - if (textPrintableCharactersLength) == None: - textPrintableCharactersLength = len(text) - - result += text - remainingSpace = lineLength - textPrintableCharactersLength - - for i in range(remainingSpace): - result += " " - - return result - - def renderHotZone(term, renderType, menu, selection, hotzoneLocation, paddingBefore = 4): - global paginationSize - selectedTextLength = len("-> ") - - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) - - if paginationStartIndex >= 1: - print(term.center("{b} {uaf} {uaf}{uaf}{uaf} {ual} {b}".format( - b=specialChars[renderMode]["borderVertical"], - uaf=specialChars[renderMode]["upArrowFull"], - ual=specialChars[renderMode]["upArrowLine"] - ))) - else: - print(term.center(commonEmptyLine(renderMode))) - - for (index, menuItem) in enumerate(menu): # Menu loop - if index >= paginationStartIndex and index < paginationStartIndex + paginationSize: - lineText = generateLineText(menuItem[0], paddingBefore=paddingBefore) - - # Menu highlight logic - if index == selection: - formattedLineText = '-> {t.blue_on_green}{title}{t.normal} <-'.format(t=term, title=menuItem[0]) - paddedLineText = generateLineText(formattedLineText, textLength=len(menuItem[0]) + selectedTextLength, paddingBefore=paddingBefore - selectedTextLength) - toPrint = paddedLineText - else: - toPrint = '{title}{t.normal}'.format(t=term, title=lineText) - # ##### - - # Menu check render logic - if menuItem[1]["checked"]: - toPrint = " (X) " + toPrint - else: - toPrint = " ( ) " + toPrint - - toPrint = "{bv} {toPrint} {bv}".format(bv=specialChars[renderMode]["borderVertical"], toPrint=toPrint) # Generate border - toPrint = term.center(toPrint) # Center Text (All lines should have the same amount of printable characters) - # ##### - print(toPrint) - - if paginationStartIndex + paginationSize < len(menu): - print(term.center("{b} {daf} {daf}{daf}{daf} {dal} {b}".format( - b=specialChars[renderMode]["borderVertical"], - daf=specialChars[renderMode]["downArrowFull"], - dal=specialChars[renderMode]["downArrowLine"] - ))) - else: - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - - - def mainRender(needsRender, menu, selection): - global paginationStartIndex - global paginationSize - term = Terminal() - - if selection >= paginationStartIndex + paginationSize: - paginationStartIndex = selection - (paginationSize - 1) + 1 - needsRender = 1 - - if selection <= paginationStartIndex - 1: - paginationStartIndex = selection - needsRender = 1 - - if needsRender == 1: - print(term.clear()) - print(term.move_y(term.height // 16)) - print(term.black_on_cornsilk4(term.center('IOTstack InfluxDB Password Options'))) - print("") - print(term.center(commonTopBorder(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Select Password Option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - - if needsRender >= 1: - renderHotZone(term, needsRender, menu, selection, hotzoneLocation) - - if needsRender == 1: - print(term.center(commonEmptyLine(renderMode))) - if not hideHelpText: - if term.height < 32: - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Not enough vertical room to render controls help text {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - else: - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Controls: {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Space] to select option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Up] and [Down] to move selection cursor {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [H] Show/hide this text {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Enter] to build and save option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Escape] to cancel changes {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonBottomBorder(renderMode))) - - def runSelection(selection): - import types - if len(mainMenuList[selection]) > 1 and isinstance(mainMenuList[selection][1], types.FunctionType): - mainMenuList[selection][1]() - else: - print(term.green_reverse('IOTstack Error: No function assigned to menu item: "{}"'.format(mainMenuList[selection][0]))) - - def isMenuItemSelectable(menu, index): - if len(menu) > index: - if len(menu[index]) > 1: - if "skip" in menu[index][1] and menu[index][1]["skip"] == True: - return False - return True - - def loadOptionsMenu(): - global mainMenuList - mainMenuList.append(["Use default database password for this build", { "checked": True }]) - mainMenuList.append(["Randomise database password for this build", { "checked": False }]) - mainMenuList.append(["Randomise database password every build", { "checked": False }]) - mainMenuList.append(["Do nothing", { "checked": False }]) - - def checkMenuItem(selection): - global mainMenuList - for (index, menuItem) in enumerate(mainMenuList): - mainMenuList[index][1]["checked"] = False - - mainMenuList[selection][1]["checked"] = True - - def saveOptions(): - try: - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - - if os.path.exists(buildSettings): - with open(r'%s' % buildSettings) as objBuildSettingsFile: - influxDbYamlBuildOptions = yaml.load(objBuildSettingsFile) - else: - influxDbYamlBuildOptions = { - "version": "1", - "application": "IOTstack", - "service": "influxdb", - "comment": "Build Settings", - } - - influxDbYamlBuildOptions["databasePasswordOption"] = "" - - for (index, menuOption) in enumerate(mainMenuList): - if menuOption[1]["checked"]: - influxDbYamlBuildOptions["databasePasswordOption"] = menuOption[0] - break - - with open(buildSettings, 'w') as outputFile: - yaml.dump(influxDbYamlBuildOptions, outputFile) - - except Exception as err: - print("Error saving InfluxDB Password options", currentServiceName) - print(err) - return False - return True - - def loadOptions(): - try: - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - - if os.path.exists(buildSettings): - with open(r'%s' % buildSettings) as objBuildSettingsFile: - influxDbYamlBuildOptions = yaml.load(objBuildSettingsFile) - - for (index, menuOption) in enumerate(mainMenuList): - if menuOption[0] == influxDbYamlBuildOptions["databasePasswordOption"]: - checkMenuItem(index) - break - - except Exception as err: - print("Error loading InfluxDB Password options", currentServiceName) - print(err) - return False - return True - - - if __name__ == 'builtins': - global signal - term = Terminal() - signal.signal(signal.SIGWINCH, onResize) - loadOptionsMenu() - loadOptions() - with term.fullscreen(): - menuNavigateDirection = 0 - mainRender(needsRender, mainMenuList, currentMenuItemIndex) - menuSelectionInProgress = True - with term.cbreak(): - while menuSelectionInProgress: - menuNavigateDirection = 0 - - if not needsRender == 0: # Only rerender when changed to prevent flickering - mainRender(needsRender, mainMenuList, currentMenuItemIndex) - needsRender = 0 - - key = term.inkey(esc_delay=0.05) - if key.is_sequence: - if key.name == 'KEY_TAB': - if paginationSize == paginationToggle[0]: - paginationSize = paginationToggle[1] - else: - paginationSize = paginationToggle[0] - mainRender(1, mainMenuList, currentMenuItemIndex) - if key.name == 'KEY_DOWN': - menuNavigateDirection += 1 - if key.name == 'KEY_UP': - menuNavigateDirection -= 1 - if key.name == 'KEY_ENTER': - if saveOptions(): - return True - else: - print("Something went wrong. Try saving the list again.") - if key.name == 'KEY_ESCAPE': - menuSelectionInProgress = False - return True - elif key: - if key == ' ': # Space pressed - checkMenuItem(currentMenuItemIndex) # Update checked list - needsRender = 2 - elif key == 'h': # H pressed - if hideHelpText: - hideHelpText = False - else: - hideHelpText = True - mainRender(1, mainMenuList, currentMenuItemIndex) - - if menuNavigateDirection != 0: # If a direction was pressed, find next selectable item - currentMenuItemIndex += menuNavigateDirection - currentMenuItemIndex = currentMenuItemIndex % len(mainMenuList) - needsRender = 2 - - while not isMenuItemSelectable(mainMenuList, currentMenuItemIndex): - currentMenuItemIndex += menuNavigateDirection - currentMenuItemIndex = currentMenuItemIndex % len(mainMenuList) - return True - - return True - -originalSignalHandler = signal.getsignal(signal.SIGINT) -main() -signal.signal(signal.SIGWINCH, originalSignalHandler) diff --git a/.templates/influxdb2/service.yml b/.templates/influxdb2/service.yml index fc0c8355f..d62c9ff65 100644 --- a/.templates/influxdb2/service.yml +++ b/.templates/influxdb2/service.yml @@ -8,7 +8,7 @@ influxdb2: - DOCKER_INFLUXDB_INIT_PASSWORD=${INFLUXDB2_PASSWORD:?eg echo INFLUXDB2_PASSWORD=mypassword >>~/IOTstack/.env} - DOCKER_INFLUXDB_INIT_ORG=${INFLUXDB2_ORG:-myorg} - DOCKER_INFLUXDB_INIT_BUCKET=${INFLUXDB2_BUCKET:-mybucket} - - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token + - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=${INFLUXDB2_ADMIN_TOKEN:-my-super-secret-auth-token} - DOCKER_INFLUXDB_INIT_MODE=setup # - DOCKER_INFLUXDB_INIT_MODE=upgrade ports: diff --git a/.templates/mariadb/build.py b/.templates/mariadb/build.py index e8893b6e9..00ecc6090 100755 --- a/.templates/mariadb/build.py +++ b/.templates/mariadb/build.py @@ -1,29 +1,24 @@ #!/usr/bin/env python3 +OPTIONS_AVAILABLE = False + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly def main(): import os - import time import sys - import ruamel.yaml import signal import subprocess from blessed import Terminal from deps.chars import specialChars, commonTopBorder, commonBottomBorder, commonEmptyLine, padText - from deps.consts import servicesDirectory, templatesDirectory, servicesFileName, buildSettingsFileName - from deps.common_functions import getExternalPorts, checkPortConflicts, generateRandomString - - yaml = ruamel.yaml.YAML() - yaml.preserve_quotes = True + from deps.consts import servicesDirectory + from deps.common_functions import getExternalPorts, checkPortConflicts global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -31,8 +26,6 @@ def main(): global serviceService serviceService = servicesDirectory + currentServiceName - serviceTemplate = templatesDirectory + currentServiceName - buildSettings = serviceService + buildSettingsFileName try: # If not already set, then set it. hideHelpText = hideHelpText @@ -44,43 +37,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -93,82 +49,7 @@ def postBuild(): # This function is optional, and will run just before the build docker-compose.yml code. def preBuild(): - # Multi-service: - with open((r'%s/' % serviceTemplate) + servicesFileName) as objServiceFile: - serviceYamlTemplate = yaml.load(objServiceFile) - - oldBuildCache = {} - try: - with open(r'%s' % buildCache) as objBuildCache: - oldBuildCache = yaml.load(objBuildCache) - except: - pass - - buildCacheServices = {} - if "services" in oldBuildCache: - buildCacheServices = oldBuildCache["services"] - - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - - if os.path.exists(buildSettings): - # Password randomisation - with open(r'%s' % buildSettings) as objBuildSettingsFile: - mariaDbYamlBuildOptions = yaml.load(objBuildSettingsFile) - if ( - mariaDbYamlBuildOptions["databasePasswordOption"] == "Randomise database password for this build" - or mariaDbYamlBuildOptions["databasePasswordOption"] == "Randomise database password every build" - or mariaDbYamlBuildOptions["databasePasswordOption"] == "Use default password for this build" - ): - if mariaDbYamlBuildOptions["databasePasswordOption"] == "Use default password for this build": - newAdminPassword = "IOtSt4ckToorMariaDb" - newPassword = "IOtSt4ckmariaDbPw" - else: - newAdminPassword = generateRandomString() - newPassword = generateRandomString() - for (index, serviceName) in enumerate(serviceYamlTemplate): - dockerComposeServicesYaml[serviceName] = serviceYamlTemplate[serviceName] - if "environment" in serviceYamlTemplate[serviceName]: - for (envIndex, envName) in enumerate(serviceYamlTemplate[serviceName]["environment"]): - envName = envName.replace("%randomAdminPassword%", newAdminPassword) - envName = envName.replace("%randomPassword%", newPassword) - dockerComposeServicesYaml[serviceName]["environment"][envIndex] = envName - - # Ensure you update the "Do nothing" and other 2 strings used for password settings in 'passwords.py' - if (mariaDbYamlBuildOptions["databasePasswordOption"] == "Randomise database password for this build"): - mariaDbYamlBuildOptions["databasePasswordOption"] = "Do nothing" - with open(buildSettings, 'w') as outputFile: - yaml.dump(mariaDbYamlBuildOptions, outputFile) - else: # Do nothing - don't change password - for (index, serviceName) in enumerate(buildCacheServices): - if serviceName in buildCacheServices: # Load service from cache if exists (to maintain password) - dockerComposeServicesYaml[serviceName] = buildCacheServices[serviceName] - else: - dockerComposeServicesYaml[serviceName] = serviceYamlTemplate[serviceName] - - else: - print("MariaDB Warning: Build settings file not found, using default password") - time.sleep(1) - newAdminPassword = "IOtSt4ckToorMariaDb" - newPassword = "IOtSt4ckmariaDbPw" - for (index, serviceName) in enumerate(serviceYamlTemplate): - dockerComposeServicesYaml[serviceName] = serviceYamlTemplate[serviceName] - if "environment" in serviceYamlTemplate[serviceName]: - for (envIndex, envName) in enumerate(serviceYamlTemplate[serviceName]["environment"]): - envName = envName.replace("%randomAdminPassword%", newAdminPassword) - envName = envName.replace("%randomPassword%", newPassword) - dockerComposeServicesYaml[serviceName]["environment"][envIndex] = envName - mariaDbYamlBuildOptions = { - "version": "1", - "application": "IOTstack", - "service": "MariaDB", - "comment": "MariaDB Build Options" - } - - mariaDbYamlBuildOptions["databasePasswordOption"] = "Do nothing" - with open(buildSettings, 'w') as outputFile: - yaml.dump(mariaDbYamlBuildOptions, outputFile) - + os.makedirs(serviceService, exist_ok=True) return True # ##################################### @@ -210,23 +91,6 @@ def goBack(): needsRender = 1 return True - def setPasswordOptions(): - global needsRender - global hasRebuiltAddons - passwordOptionsMenuFilePath = "./.templates/{currentService}/passwords.py".format(currentService=currentServiceName) - with open(passwordOptionsMenuFilePath, "rb") as pythonDynamicImportFile: - code = compile(pythonDynamicImportFile.read(), passwordOptionsMenuFilePath, "exec") - execGlobals = { - "currentServiceName": currentServiceName, - "renderMode": renderMode - } - execLocals = {} - screenActive = False - exec(code, execGlobals, execLocals) - signal.signal(signal.SIGWINCH, onResize) - screenActive = True - needsRender = 1 - def onResize(sig, action): global mariaDbBuildOptions global currentMenuItemIndex @@ -239,21 +103,22 @@ def createMenu(): global serviceService mariaDbBuildOptions = [] - mariaDbBuildOptions.append([ - "MariaDB Password Options", - setPasswordOptions - ]) mariaDbBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -336,6 +201,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, mariaDbBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -382,17 +248,50 @@ def menuEntryPoint(): #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass - -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'mariadb': - main() -else: - print("Error. '{}' Tried to run 'mariadb' config".format(currentServiceName)) + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + + +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/mariadb/passwords.py b/.templates/mariadb/passwords.py deleted file mode 100755 index 92f38fcf1..000000000 --- a/.templates/mariadb/passwords.py +++ /dev/null @@ -1,326 +0,0 @@ -#!/usr/bin/env python3 - -import signal - -def main(): - from blessed import Terminal - from deps.chars import specialChars, commonTopBorder, commonBottomBorder, commonEmptyLine - from deps.consts import servicesDirectory, templatesDirectory, buildSettingsFileName - import time - import subprocess - import ruamel.yaml - import os - - global signal - global currentServiceName - global menuSelectionInProgress - global mainMenuList - global currentMenuItemIndex - global renderMode - global paginationSize - global paginationStartIndex - global hideHelpText - - yaml = ruamel.yaml.YAML() - yaml.preserve_quotes = True - - try: # If not already set, then set it. - hideHelpText = hideHelpText - except: - hideHelpText = False - - term = Terminal() - hotzoneLocation = [((term.height // 16) + 6), 0] - paginationToggle = [10, term.height - 25] - paginationStartIndex = 0 - paginationSize = paginationToggle[0] - - serviceService = servicesDirectory + currentServiceName - serviceTemplate = templatesDirectory + currentServiceName - buildSettings = serviceService + buildSettingsFileName - - def goBack(): - global menuSelectionInProgress - global needsRender - menuSelectionInProgress = False - needsRender = 1 - return True - - mainMenuList = [] - - hotzoneLocation = [((term.height // 16) + 6), 0] - - menuSelectionInProgress = True - currentMenuItemIndex = 0 - menuNavigateDirection = 0 - - # Render Modes: - # 0 = No render needed - # 1 = Full render - # 2 = Hotzone only - needsRender = 1 - - def onResize(sig, action): - global mainMenuList - global currentMenuItemIndex - mainRender(1, mainMenuList, currentMenuItemIndex) - - def generateLineText(text, textLength=None, paddingBefore=0, lineLength=64): - result = "" - for i in range(paddingBefore): - result += " " - - textPrintableCharactersLength = textLength - - if (textPrintableCharactersLength) == None: - textPrintableCharactersLength = len(text) - - result += text - remainingSpace = lineLength - textPrintableCharactersLength - - for i in range(remainingSpace): - result += " " - - return result - - def renderHotZone(term, renderType, menu, selection, hotzoneLocation, paddingBefore = 4): - global paginationSize - selectedTextLength = len("-> ") - - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) - - if paginationStartIndex >= 1: - print(term.center("{b} {uaf} {uaf}{uaf}{uaf} {ual} {b}".format( - b=specialChars[renderMode]["borderVertical"], - uaf=specialChars[renderMode]["upArrowFull"], - ual=specialChars[renderMode]["upArrowLine"] - ))) - else: - print(term.center(commonEmptyLine(renderMode))) - - for (index, menuItem) in enumerate(menu): # Menu loop - if index >= paginationStartIndex and index < paginationStartIndex + paginationSize: - lineText = generateLineText(menuItem[0], paddingBefore=paddingBefore) - - # Menu highlight logic - if index == selection: - formattedLineText = '-> {t.blue_on_green}{title}{t.normal} <-'.format(t=term, title=menuItem[0]) - paddedLineText = generateLineText(formattedLineText, textLength=len(menuItem[0]) + selectedTextLength, paddingBefore=paddingBefore - selectedTextLength) - toPrint = paddedLineText - else: - toPrint = '{title}{t.normal}'.format(t=term, title=lineText) - # ##### - - # Menu check render logic - if menuItem[1]["checked"]: - toPrint = " (X) " + toPrint - else: - toPrint = " ( ) " + toPrint - - toPrint = "{bv} {toPrint} {bv}".format(bv=specialChars[renderMode]["borderVertical"], toPrint=toPrint) # Generate border - toPrint = term.center(toPrint) # Center Text (All lines should have the same amount of printable characters) - # ##### - print(toPrint) - - if paginationStartIndex + paginationSize < len(menu): - print(term.center("{b} {daf} {daf}{daf}{daf} {dal} {b}".format( - b=specialChars[renderMode]["borderVertical"], - daf=specialChars[renderMode]["downArrowFull"], - dal=specialChars[renderMode]["downArrowLine"] - ))) - else: - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - - - def mainRender(needsRender, menu, selection): - global paginationStartIndex - global paginationSize - term = Terminal() - - if selection >= paginationStartIndex + paginationSize: - paginationStartIndex = selection - (paginationSize - 1) + 1 - needsRender = 1 - - if selection <= paginationStartIndex - 1: - paginationStartIndex = selection - needsRender = 1 - - if needsRender == 1: - print(term.clear()) - print(term.move_y(term.height // 16)) - print(term.black_on_cornsilk4(term.center('IOTstack MariaDB Password Options'))) - print("") - print(term.center(commonTopBorder(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Select Password Option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - - if needsRender >= 1: - renderHotZone(term, needsRender, menu, selection, hotzoneLocation) - - if needsRender == 1: - print(term.center(commonEmptyLine(renderMode))) - if not hideHelpText: - if term.height < 32: - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Not enough vertical room to render controls help text {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - else: - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Controls: {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Space] to select option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Up] and [Down] to move selection cursor {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [H] Show/hide this text {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Enter] to build and save option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Escape] to cancel changes {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonBottomBorder(renderMode))) - - def runSelection(selection): - import types - if len(mainMenuList[selection]) > 1 and isinstance(mainMenuList[selection][1], types.FunctionType): - mainMenuList[selection][1]() - else: - print(term.green_reverse('IOTstack Error: No function assigned to menu item: "{}"'.format(mainMenuList[selection][0]))) - - def isMenuItemSelectable(menu, index): - if len(menu) > index: - if len(menu[index]) > 1: - if "skip" in menu[index][1] and menu[index][1]["skip"] == True: - return False - return True - - def loadOptionsMenu(): - global mainMenuList - mainMenuList.append(["Use default password for this build", { "checked": True }]) - mainMenuList.append(["Randomise database password for this build", { "checked": False }]) - mainMenuList.append(["Randomise database password every build", { "checked": False }]) - mainMenuList.append(["Do nothing", { "checked": False }]) - - def checkMenuItem(selection): - global mainMenuList - for (index, menuItem) in enumerate(mainMenuList): - mainMenuList[index][1]["checked"] = False - - mainMenuList[selection][1]["checked"] = True - - def saveOptions(): - try: - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - - if os.path.exists(buildSettings): - with open(r'%s' % buildSettings) as objBuildSettingsFile: - mariaDbYamlBuildOptions = yaml.load(objBuildSettingsFile) - else: - mariaDbYamlBuildOptions = { - "version": "1", - "application": "IOTstack", - "service": "mariadb", - "comment": "Build Settings", - } - - mariaDbYamlBuildOptions["databasePasswordOption"] = "" - - for (index, menuOption) in enumerate(mainMenuList): - if menuOption[1]["checked"]: - mariaDbYamlBuildOptions["databasePasswordOption"] = menuOption[0] - break - - with open(buildSettings, 'w') as outputFile: - yaml.dump(mariaDbYamlBuildOptions, outputFile) - - except Exception as err: - print("Error saving MariaDB Password options", currentServiceName) - print(err) - return False - return True - - def loadOptions(): - try: - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - - if os.path.exists(buildSettings): - with open(r'%s' % buildSettings) as objBuildSettingsFile: - mariaDbYamlBuildOptions = yaml.load(objBuildSettingsFile) - - for (index, menuOption) in enumerate(mainMenuList): - if menuOption[0] == mariaDbYamlBuildOptions["databasePasswordOption"]: - checkMenuItem(index) - break - - except Exception as err: - print("Error loading MariaDB Password options", currentServiceName) - print(err) - return False - return True - - - if __name__ == 'builtins': - global signal - term = Terminal() - signal.signal(signal.SIGWINCH, onResize) - loadOptionsMenu() - loadOptions() - with term.fullscreen(): - menuNavigateDirection = 0 - mainRender(needsRender, mainMenuList, currentMenuItemIndex) - menuSelectionInProgress = True - with term.cbreak(): - while menuSelectionInProgress: - menuNavigateDirection = 0 - - if not needsRender == 0: # Only rerender when changed to prevent flickering - mainRender(needsRender, mainMenuList, currentMenuItemIndex) - needsRender = 0 - - key = term.inkey(esc_delay=0.05) - if key.is_sequence: - if key.name == 'KEY_TAB': - if paginationSize == paginationToggle[0]: - paginationSize = paginationToggle[1] - else: - paginationSize = paginationToggle[0] - mainRender(1, mainMenuList, currentMenuItemIndex) - if key.name == 'KEY_DOWN': - menuNavigateDirection += 1 - if key.name == 'KEY_UP': - menuNavigateDirection -= 1 - if key.name == 'KEY_ENTER': - if saveOptions(): - return True - else: - print("Something went wrong. Try saving the list again.") - if key.name == 'KEY_ESCAPE': - menuSelectionInProgress = False - return True - elif key: - if key == ' ': # Space pressed - checkMenuItem(currentMenuItemIndex) # Update checked list - needsRender = 2 - elif key == 'h': # H pressed - if hideHelpText: - hideHelpText = False - else: - hideHelpText = True - mainRender(1, mainMenuList, currentMenuItemIndex) - - if menuNavigateDirection != 0: # If a direction was pressed, find next selectable item - currentMenuItemIndex += menuNavigateDirection - currentMenuItemIndex = currentMenuItemIndex % len(mainMenuList) - needsRender = 2 - - while not isMenuItemSelectable(mainMenuList, currentMenuItemIndex): - currentMenuItemIndex += menuNavigateDirection - currentMenuItemIndex = currentMenuItemIndex % len(mainMenuList) - return True - - return True - -originalSignalHandler = signal.getsignal(signal.SIGINT) -main() -signal.signal(signal.SIGWINCH, originalSignalHandler) diff --git a/.templates/mariadb/service.yml b/.templates/mariadb/service.yml index f1dad69c6..03c7e135b 100644 --- a/.templates/mariadb/service.yml +++ b/.templates/mariadb/service.yml @@ -5,10 +5,10 @@ mariadb: - TZ=${TZ:-Etc/UTC} - PUID=1000 - PGID=1000 - - MYSQL_ROOT_PASSWORD=${MARIADB_ROOT_PASSWORD:?eg echo MARIADB_ROOT_PASSWORD=%randomAdminPassword% >>~/IOTstack/.env} + - MYSQL_ROOT_PASSWORD=${MARIADB_ROOT_PASSWORD:?eg echo MARIADB_ROOT_PASSWORD=IOtSt4ckToorMariaDb >>~/IOTstack/.env} - MYSQL_DATABASE=${MARIADB_DATABASE:-default} - MYSQL_USER=${MARIADB_USER:-mariadbuser} - - MYSQL_PASSWORD=${MARIADB_USER_PASSWORD:?eg echo MARIADB_USER_PASSWORD=%randomPassword% >>~/IOTstack/.env} + - MYSQL_PASSWORD=${MARIADB_USER_PASSWORD:?eg echo MARIADB_USER_PASSWORD=IOtSt4ckmariaDbPw >>~/IOTstack/.env} volumes: - ./volumes/mariadb/config:/config - ./volumes/mariadb/db_backup:/backup diff --git a/.templates/mjpg-streamer/service.yml b/.templates/mjpg-streamer/service.yml index 3e9f9a006..420bc3f12 100644 --- a/.templates/mjpg-streamer/service.yml +++ b/.templates/mjpg-streamer/service.yml @@ -4,8 +4,8 @@ mjpg-streamer: restart: unless-stopped environment: - TZ=${TZ:-Etc/UTC} - - MJPG_STREAMER_USERNAME=${MJPG_STREAMER_USERNAME:-} - - MJPG_STREAMER_PASSWORD=${MJPG_STREAMER_PASSWORD:-} + - MJPG_STREAMER_USERNAME=${MJPG_STREAMER_USERNAME:-iotstack} + - MJPG_STREAMER_PASSWORD=${MJPG_STREAMER_PASSWORD:-IOtSt4ckMJPG} - MJPG_STREAMER_SIZE=${MJPG_STREAMER_SIZE:-} - MJPG_STREAMER_FPS=${MJPG_STREAMER_FPS:-} ports: diff --git a/.templates/motioneye/build.py b/.templates/motioneye/build.py index 5d46dbae5..13a6f876d 100755 --- a/.templates/motioneye/build.py +++ b/.templates/motioneye/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -22,7 +22,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -42,43 +41,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -181,13 +143,18 @@ def createMenu(): motionEyeBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -270,6 +237,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, motionEyeBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -316,17 +284,50 @@ def menuEntryPoint(): #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'motioneye': - main() -else: - print("Error. '{}' Tried to run 'motioneye' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/n8n/build.py b/.templates/n8n/build.py index d717b0008..44c49a877 100755 --- a/.templates/n8n/build.py +++ b/.templates/n8n/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -22,7 +22,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -42,43 +41,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -177,13 +139,18 @@ def createMenu(): n8nBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -266,6 +233,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, n8nBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -312,17 +280,50 @@ def menuEntryPoint(): #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'n8n': - main() -else: - print("Error. '{}' Tried to run 'n8n' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/nextcloud/build.py b/.templates/nextcloud/build.py index cfb6bb54a..179be604c 100755 --- a/.templates/nextcloud/build.py +++ b/.templates/nextcloud/build.py @@ -1,29 +1,23 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly def main(): import os - import time - import ruamel.yaml import signal import sys import subprocess from blessed import Terminal from deps.chars import specialChars, commonTopBorder, commonBottomBorder, commonEmptyLine, padText - from deps.consts import servicesDirectory, templatesDirectory, volumesDirectory, buildSettingsFileName, buildCache, servicesFileName - from deps.common_functions import getExternalPorts, getInternalPorts, checkPortConflicts, enterPortNumberWithWhiptail, generateRandomString - - yaml = ruamel.yaml.YAML() - yaml.preserve_quotes = True + from deps.consts import servicesDirectory, volumesDirectory + from deps.common_functions import getExternalPorts, getInternalPorts, checkPortConflicts, enterPortNumberWithWhiptail global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -32,8 +26,6 @@ def main(): serviceVolume = volumesDirectory + currentServiceName serviceService = servicesDirectory + currentServiceName - serviceTemplate = templatesDirectory + currentServiceName - buildSettings = serviceService + buildSettingsFileName try: # If not already set, then set it. hideHelpText = hideHelpText @@ -45,43 +37,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -90,97 +45,15 @@ def runChecks(): # This function is optional, and will run after the docker-compose.yml file is written to disk. def postBuild(): - commandToRun = "chmod -R 0770 %s" % serviceVolume + '/html' - print('[Nextcloud::postBuild]: %s' % commandToRun) - subprocess.call(commandToRun, shell=True) - return True + commandToRun = ["chmod", "-R", "0770", serviceVolume + "/html"] + print('[Nextcloud::postBuild]: %s' % " ".join(commandToRun)) + return subprocess.call(commandToRun) == 0 # This function is optional, and will run just before the build docker-compose.yml code. def preBuild(): - global dockerComposeServicesYaml - # Setup service directory - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - + os.makedirs(serviceService, exist_ok=True) os.makedirs(serviceVolume, exist_ok=True) - os.makedirs(serviceVolume + '/html', exist_ok=True) - - # Multi-service: - with open((r'%s/' % serviceTemplate) + servicesFileName) as objServiceFile: - servicesListed = yaml.load(objServiceFile) - - oldBuildCache = {} - try: - with open(r'%s' % buildCache) as objBuildCache: - oldBuildCache = yaml.load(objBuildCache) - except: - pass - - buildCacheServices = {} - if "services" in oldBuildCache: - buildCacheServices = oldBuildCache["services"] - - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - - if os.path.exists(buildSettings): - - # Password randomisation - with open(r'%s' % buildSettings) as objBuildSettingsFile: - nextCloudYamlBuildOptions = yaml.load(objBuildSettingsFile) - if ( - nextCloudYamlBuildOptions["databasePasswordOption"] == "Randomise passwords for this build" - or nextCloudYamlBuildOptions["databasePasswordOption"] == "Randomise passwords every build" - or nextCloudYamlBuildOptions["databasePasswordOption"] == "Use default passwords for this build" - ): - if nextCloudYamlBuildOptions["databasePasswordOption"] == "Use default passwords for this build": - mySqlRootPassword = "IOtSt4ckToorMySqlDb" - mySqlPassword = "IOtSt4ckmySqlDbPw" - else: - mySqlPassword = generateRandomString() - mySqlRootPassword = generateRandomString() - - for (index, serviceName) in enumerate(servicesListed): - dockerComposeServicesYaml[serviceName] = servicesListed[serviceName] - if "environment" in servicesListed[serviceName]: - for (envIndex, envName) in enumerate(servicesListed[serviceName]["environment"]): - envName = envName.replace("%randomMySqlPassword%", mySqlPassword) - dockerComposeServicesYaml[serviceName]["environment"][envIndex] = envName.replace("%randomPassword%", mySqlRootPassword) - - # Ensure you update the "Do nothing" and other 2 strings used for password settings in 'passwords.py' - if (nextCloudYamlBuildOptions["databasePasswordOption"] == "Randomise passwords for this build"): - nextCloudYamlBuildOptions["databasePasswordOption"] = "Do nothing" - with open(buildSettings, 'w') as outputFile: - yaml.dump(nextCloudYamlBuildOptions, outputFile) - else: # Do nothing - don't change password - for (index, serviceName) in enumerate(buildCacheServices): - if serviceName in buildCacheServices: # Load service from cache if exists (to maintain password) - dockerComposeServicesYaml[serviceName] = buildCacheServices[serviceName] - else: - dockerComposeServicesYaml[serviceName] = servicesListed[serviceName] - - else: - print("NextCloud Warning: Build settings file not found, using default password") - time.sleep(1) - mySqlRootPassword = "IOtSt4ckToorMySqlDb" - mySqlPassword = "IOtSt4ckmySqlDbPw" - for (index, serviceName) in enumerate(servicesListed): - dockerComposeServicesYaml[serviceName] = servicesListed[serviceName] - if "environment" in servicesListed[serviceName]: - for (envIndex, envName) in enumerate(servicesListed[serviceName]["environment"]): - envName = envName.replace("%randomMySqlPassword%", mySqlPassword) - dockerComposeServicesYaml[serviceName]["environment"][envIndex] = envName.replace("%randomPassword%", mySqlRootPassword) - nextCloudYamlBuildOptions = { - "version": "1", - "application": "IOTstack", - "service": "NextCloud", - "comment": "NextCloud Build Options" - } - - nextCloudYamlBuildOptions["databasePasswordOption"] = "Do nothing" - with open(buildSettings, 'w') as outputFile: - yaml.dump(nextCloudYamlBuildOptions, outputFile) - + os.makedirs(serviceVolume + "/html", exist_ok=True) return True # ##################################### @@ -238,23 +111,6 @@ def enterPortNumberExec(): createMenu() needsRender = 1 - def setPasswordOptions(): - global needsRender - global hasRebuiltAddons - passwordOptionsMenuFilePath = "./.templates/{currentService}/passwords.py".format(currentService=currentServiceName) - with open(passwordOptionsMenuFilePath, "rb") as pythonDynamicImportFile: - code = compile(pythonDynamicImportFile.read(), passwordOptionsMenuFilePath, "exec") - execGlobals = { - "currentServiceName": currentServiceName, - "renderMode": renderMode - } - execLocals = {} - screenActive = False - exec(code, execGlobals, execLocals) - signal.signal(signal.SIGWINCH, onResize) - screenActive = True - needsRender = 1 - def onResize(sig, action): global nextCloudBuildOptions global currentMenuItemIndex @@ -273,20 +129,21 @@ def createMenu(): ]) except: # Error getting port pass - nextCloudBuildOptions.append([ - "Database Password Options", - setPasswordOptions - ]) nextCloudBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -369,6 +226,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, nextCloudBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -415,17 +273,50 @@ def menuEntryPoint(): #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'nextcloud': - main() -else: - print("Error. '{}' Tried to run 'nextcloud' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/nextcloud/passwords.py b/.templates/nextcloud/passwords.py deleted file mode 100755 index e54e09063..000000000 --- a/.templates/nextcloud/passwords.py +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env python3 - -import signal - -def main(): - from blessed import Terminal - from deps.chars import specialChars, commonTopBorder, commonBottomBorder, commonEmptyLine - from deps.consts import servicesDirectory, templatesDirectory, buildSettingsFileName - import time - import subprocess - import ruamel.yaml - import os - - global signal - global currentServiceName - global menuSelectionInProgress - global mainMenuList - global currentMenuItemIndex - global renderMode - global paginationSize - global paginationStartIndex - global hideHelpText - - yaml = ruamel.yaml.YAML() - yaml.preserve_quotes = True - - try: # If not already set, then set it. - hideHelpText = hideHelpText - except: - hideHelpText = False - - term = Terminal() - hotzoneLocation = [((term.height // 16) + 6), 0] - paginationToggle = [10, term.height - 25] - paginationStartIndex = 0 - paginationSize = paginationToggle[0] - - serviceService = servicesDirectory + currentServiceName - serviceTemplate = templatesDirectory + currentServiceName - buildSettings = serviceService + buildSettingsFileName - - def goBack(): - global menuSelectionInProgress - global needsRender - menuSelectionInProgress = False - needsRender = 1 - return True - - mainMenuList = [] - - hotzoneLocation = [((term.height // 16) + 6), 0] - - menuSelectionInProgress = True - currentMenuItemIndex = 0 - menuNavigateDirection = 0 - - # Render Modes: - # 0 = No render needed - # 1 = Full render - # 2 = Hotzone only - needsRender = 1 - - def onResize(sig, action): - global mainMenuList - global currentMenuItemIndex - mainRender(1, mainMenuList, currentMenuItemIndex) - - def generateLineText(text, textLength=None, paddingBefore=0, lineLength=64): - result = "" - for i in range(paddingBefore): - result += " " - - textPrintableCharactersLength = textLength - - if (textPrintableCharactersLength) == None: - textPrintableCharactersLength = len(text) - - result += text - remainingSpace = lineLength - textPrintableCharactersLength - - for i in range(remainingSpace): - result += " " - - return result - - def renderHotZone(term, renderType, menu, selection, hotzoneLocation, paddingBefore = 4): - global paginationSize - selectedTextLength = len("-> ") - - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) - - if paginationStartIndex >= 1: - print(term.center("{b} {uaf} {uaf}{uaf}{uaf} {ual} {b}".format( - b=specialChars[renderMode]["borderVertical"], - uaf=specialChars[renderMode]["upArrowFull"], - ual=specialChars[renderMode]["upArrowLine"] - ))) - else: - print(term.center(commonEmptyLine(renderMode))) - - for (index, menuItem) in enumerate(menu): # Menu loop - if index >= paginationStartIndex and index < paginationStartIndex + paginationSize: - lineText = generateLineText(menuItem[0], paddingBefore=paddingBefore) - - # Menu highlight logic - if index == selection: - formattedLineText = '-> {t.blue_on_green}{title}{t.normal} <-'.format(t=term, title=menuItem[0]) - paddedLineText = generateLineText(formattedLineText, textLength=len(menuItem[0]) + selectedTextLength, paddingBefore=paddingBefore - selectedTextLength) - toPrint = paddedLineText - else: - toPrint = '{title}{t.normal}'.format(t=term, title=lineText) - # ##### - - # Menu check render logic - if menuItem[1]["checked"]: - toPrint = " (X) " + toPrint - else: - toPrint = " ( ) " + toPrint - - toPrint = "{bv} {toPrint} {bv}".format(bv=specialChars[renderMode]["borderVertical"], toPrint=toPrint) # Generate border - toPrint = term.center(toPrint) # Center Text (All lines should have the same amount of printable characters) - # ##### - print(toPrint) - - if paginationStartIndex + paginationSize < len(menu): - print(term.center("{b} {daf} {daf}{daf}{daf} {dal} {b}".format( - b=specialChars[renderMode]["borderVertical"], - daf=specialChars[renderMode]["downArrowFull"], - dal=specialChars[renderMode]["downArrowLine"] - ))) - else: - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - - - def mainRender(needsRender, menu, selection): - global paginationStartIndex - global paginationSize - term = Terminal() - - if selection >= paginationStartIndex + paginationSize: - paginationStartIndex = selection - (paginationSize - 1) + 1 - needsRender = 1 - - if selection <= paginationStartIndex - 1: - paginationStartIndex = selection - needsRender = 1 - - if needsRender == 1: - print(term.clear()) - print(term.move_y(term.height // 16)) - print(term.black_on_cornsilk4(term.center('IOTstack NextCloud Password Options'))) - print("") - print(term.center(commonTopBorder(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Select Password Option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - - if needsRender >= 1: - renderHotZone(term, needsRender, menu, selection, hotzoneLocation) - - if needsRender == 1: - print(term.center(commonEmptyLine(renderMode))) - if not hideHelpText: - if term.height < 32: - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Not enough vertical room to render controls help text {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - else: - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Controls: {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Space] to select option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Up] and [Down] to move selection cursor {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [H] Show/hide this text {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Enter] to build and save option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Escape] to cancel changes {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonBottomBorder(renderMode))) - - def runSelection(selection): - import types - if len(mainMenuList[selection]) > 1 and isinstance(mainMenuList[selection][1], types.FunctionType): - mainMenuList[selection][1]() - else: - print(term.green_reverse('IOTstack Error: No function assigned to menu item: "{}"'.format(mainMenuList[selection][0]))) - - def isMenuItemSelectable(menu, index): - if len(menu) > index: - if len(menu[index]) > 1: - if "skip" in menu[index][1] and menu[index][1]["skip"] == True: - return False - return True - - def loadOptionsMenu(): - global mainMenuList - mainMenuList.append(["Use default passwords for this build", { "checked": True }]) - mainMenuList.append(["Randomise passwords for this build", { "checked": False }]) - mainMenuList.append(["Randomise passwords every build", { "checked": False }]) - mainMenuList.append(["Do nothing", { "checked": False }]) - - def checkMenuItem(selection): - global mainMenuList - for (index, menuItem) in enumerate(mainMenuList): - mainMenuList[index][1]["checked"] = False - - mainMenuList[selection][1]["checked"] = True - - def saveOptions(): - try: - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - - if os.path.exists(buildSettings): - with open(r'%s' % buildSettings) as objBuildSettingsFile: - nextCloudYamlBuildOptions = yaml.load(objBuildSettingsFile) - else: - nextCloudYamlBuildOptions = { - "version": "1", - "application": "IOTstack", - "service": "NextCloud", - "comment": "NextCloud Build Options" - } - - nextCloudYamlBuildOptions["databasePasswordOption"] = "" - - for (index, menuOption) in enumerate(mainMenuList): - if menuOption[1]["checked"]: - nextCloudYamlBuildOptions["databasePasswordOption"] = menuOption[0] - break - - with open(buildSettings, 'w') as outputFile: - yaml.dump(nextCloudYamlBuildOptions, outputFile) - - except Exception as err: - print("Error saving NextCloud Password options", currentServiceName) - print(err) - return False - global hasRebuiltHardwareSelection - hasRebuiltHardwareSelection = True - return True - - def loadOptions(): - try: - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - - if os.path.exists(buildSettings): - with open(r'%s' % buildSettings) as objBuildSettingsFile: - nextCloudYamlBuildOptions = yaml.load(objBuildSettingsFile) - - for (index, menuOption) in enumerate(mainMenuList): - if menuOption[0] == nextCloudYamlBuildOptions["databasePasswordOption"]: - checkMenuItem(index) - break - - except Exception as err: - print("Error loading NextCloud Password options", currentServiceName) - print(err) - return False - return True - - - if __name__ == 'builtins': - global signal - term = Terminal() - signal.signal(signal.SIGWINCH, onResize) - loadOptionsMenu() - loadOptions() - with term.fullscreen(): - menuNavigateDirection = 0 - mainRender(needsRender, mainMenuList, currentMenuItemIndex) - menuSelectionInProgress = True - with term.cbreak(): - while menuSelectionInProgress: - menuNavigateDirection = 0 - - if not needsRender == 0: # Only rerender when changed to prevent flickering - mainRender(needsRender, mainMenuList, currentMenuItemIndex) - needsRender = 0 - - key = term.inkey(esc_delay=0.05) - if key.is_sequence: - if key.name == 'KEY_TAB': - if paginationSize == paginationToggle[0]: - paginationSize = paginationToggle[1] - else: - paginationSize = paginationToggle[0] - mainRender(1, mainMenuList, currentMenuItemIndex) - if key.name == 'KEY_DOWN': - menuNavigateDirection += 1 - if key.name == 'KEY_UP': - menuNavigateDirection -= 1 - if key.name == 'KEY_ENTER': - if saveOptions(): - return True - else: - print("Something went wrong. Try saving the list again.") - if key.name == 'KEY_ESCAPE': - menuSelectionInProgress = False - return True - elif key: - if key == ' ': # Space pressed - checkMenuItem(currentMenuItemIndex) # Update checked list - needsRender = 2 - elif key == 'h': # H pressed - if hideHelpText: - hideHelpText = False - else: - hideHelpText = True - mainRender(1, mainMenuList, currentMenuItemIndex) - - if menuNavigateDirection != 0: # If a direction was pressed, find next selectable item - currentMenuItemIndex += menuNavigateDirection - currentMenuItemIndex = currentMenuItemIndex % len(mainMenuList) - needsRender = 2 - - while not isMenuItemSelectable(mainMenuList, currentMenuItemIndex): - currentMenuItemIndex += menuNavigateDirection - currentMenuItemIndex = currentMenuItemIndex % len(mainMenuList) - return True - - return True - -originalSignalHandler = signal.getsignal(signal.SIGINT) -main() -signal.signal(signal.SIGWINCH, originalSignalHandler) diff --git a/.templates/nextcloud/service.yml b/.templates/nextcloud/service.yml index 1b462bb31..b7f87a4bc 100644 --- a/.templates/nextcloud/service.yml +++ b/.templates/nextcloud/service.yml @@ -5,7 +5,7 @@ nextcloud: environment: - TZ=${TZ:-Etc/UTC} - MYSQL_HOST=nextcloud_db - - MYSQL_PASSWORD=${NEXTCLOUD_DB_USER_PASSWORD:?eg echo NEXTCLOUD_DB_USER_PASSWORD=%randomMySqlPassword% >>~/IOTstack/.env} + - MYSQL_PASSWORD=${NEXTCLOUD_DB_USER_PASSWORD:?eg echo NEXTCLOUD_DB_USER_PASSWORD=IOtSt4ckmySqlDbPw >>~/IOTstack/.env} - MYSQL_DATABASE=${NEXTCLOUD_DB_NAME:-nextcloud} - MYSQL_USER=${NEXTCLOUD_DB_USER:-nextcloud} ports: @@ -27,8 +27,8 @@ nextcloud_db: - TZ=${TZ:-Etc/UTC} - PUID=1000 - PGID=1000 - - MYSQL_ROOT_PASSWORD=${NEXTCLOUD_DB_ROOT_PASSWORD:?eg echo NEXTCLOUD_DB_ROOT_PASSWORD=%randomPassword% >>~/IOTstack/.env} - - MYSQL_PASSWORD=${NEXTCLOUD_DB_USER_PASSWORD:?eg echo NEXTCLOUD_DB_USER_PASSWORD=%randomMySqlPassword% >>~/IOTstack/.env} + - MYSQL_ROOT_PASSWORD=${NEXTCLOUD_DB_ROOT_PASSWORD:?eg echo NEXTCLOUD_DB_ROOT_PASSWORD=IOtSt4ckToorMySqlDb >>~/IOTstack/.env} + - MYSQL_PASSWORD=${NEXTCLOUD_DB_USER_PASSWORD:?eg echo NEXTCLOUD_DB_USER_PASSWORD=IOtSt4ckmySqlDbPw >>~/IOTstack/.env} - MYSQL_DATABASE=${NEXTCLOUD_DB_NAME:-nextcloud} - MYSQL_USER=${NEXTCLOUD_DB_USER:-nextcloud} volumes: diff --git a/.templates/nodered/addons.py b/.templates/nodered/addons.py index 75571382c..7934d97a2 100755 --- a/.templates/nodered/addons.py +++ b/.templates/nodered/addons.py @@ -90,7 +90,7 @@ def renderHotZone(term, renderType, menu, selection, hotzoneLocation, paddingBef global paginationSize selectedTextLength = len("-> ") - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") if paginationStartIndex >= 1: print(term.center("{b} {uaf} {uaf}{uaf}{uaf} {ual} {b}".format( @@ -284,6 +284,7 @@ def saveAddonList(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, mainMenuList, currentMenuItemIndex) + needsRender = 0 dockerCommandsSelectionInProgress = True with term.cbreak(): while dockerCommandsSelectionInProgress: @@ -344,6 +345,6 @@ def saveAddonList(): return True -originalSignalHandler = signal.getsignal(signal.SIGINT) +originalSignalHandler = signal.getsignal(signal.SIGWINCH) main() signal.signal(signal.SIGWINCH, originalSignalHandler) diff --git a/.templates/nodered/build.py b/.templates/nodered/build.py index 5da092c22..16cda709e 100755 --- a/.templates/nodered/build.py +++ b/.templates/nodered/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -19,7 +19,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global renderMode # For rendering fancy or basic ascii characters global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -49,44 +48,6 @@ def main(): dockerfileTemplateReplace = "%run npm install modules list%" - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - global buildHooks - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -246,13 +207,18 @@ def createMenu(): nodeRedBuildOptions.insert(0, ["Select & build addons list", selectNodeRedAddons]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -343,6 +309,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, nodeRedBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -388,17 +355,50 @@ def menuEntryPoint(): # End menu section #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'nodered': - main() -else: - print("Error. '{}' Tried to run 'nodered' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/openhab/build.py b/.templates/openhab/build.py index fd9fd3932..01520f769 100755 --- a/.templates/openhab/build.py +++ b/.templates/openhab/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -9,48 +9,10 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -74,17 +36,45 @@ def preBuild(): # End Supporting functions # ##################################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass - -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'openhab': - main() -else: - print("Error. '{}' Tried to run 'openhab' config".format(currentServiceName)) + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + + +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/openhab/service.yml b/.templates/openhab/service.yml index b2021a37b..5713b473d 100644 --- a/.templates/openhab/service.yml +++ b/.templates/openhab/service.yml @@ -8,10 +8,10 @@ openhab: - "./volumes/openhab/conf:/openhab/conf" - "./volumes/openhab/userdata:/openhab/userdata" environment: - - TZ: ${TZ:-Etc/UTC} - - OPENHAB_HTTP_PORT: 4050 - - OPENHAB_HTTPS_PORT: 4051 - - EXTRA_JAVA_OPTS: -Duser.timezone=${TZ:-Etc/UTC} + TZ: ${TZ:-Etc/UTC} + OPENHAB_HTTP_PORT: "4050" + OPENHAB_HTTPS_PORT: "4051" + EXTRA_JAVA_OPTS: -Duser.timezone=${TZ:-Etc/UTC} x-logging: options: max-size: "5m" diff --git a/.templates/otbr/build.py b/.templates/otbr/build.py index cecdf58c6..d4d63720a 100755 --- a/.templates/otbr/build.py +++ b/.templates/otbr/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -22,7 +22,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -46,43 +45,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -97,6 +59,9 @@ def postBuild(): def preBuild(): global dockerComposeServicesYaml global currentServiceName + if not os.path.exists(buildSettings): + print("OTBR hardware is not configured. Select hardware from Options before building.") + return False with open("{serviceDir}{buildSettings}".format(serviceDir=serviceService, buildSettings=buildSettingsFileName)) as objHardwareListFile: otbrYamlBuildOptions = yaml.load(objHardwareListFile) @@ -232,13 +197,18 @@ def createMenu(): threadBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -334,6 +304,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, threadBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -376,17 +347,50 @@ def menuEntryPoint(): return True + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'otbr': - main() -else: - print("Error. '{}' Tried to run 'otbr' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/otbr/select_hardware.py b/.templates/otbr/select_hardware.py index 08d3c49dd..99076b3b6 100755 --- a/.templates/otbr/select_hardware.py +++ b/.templates/otbr/select_hardware.py @@ -90,7 +90,7 @@ def renderHotZone(term, renderType, menu, selection, hotzoneLocation, paddingBef global paginationSize selectedTextLength = len("-> ") - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") if paginationStartIndex >= 1: print(term.center("{b} {uaf} {uaf}{uaf}{uaf} {ual} {b}".format( @@ -280,6 +280,7 @@ def saveAddonList(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, mainMenuList, currentMenuItemIndex) + needsRender = 0 dockerCommandsSelectionInProgress = True with term.cbreak(): while dockerCommandsSelectionInProgress: @@ -332,6 +333,6 @@ def saveAddonList(): return True -originalSignalHandler = signal.getsignal(signal.SIGINT) +originalSignalHandler = signal.getsignal(signal.SIGWINCH) main() signal.signal(signal.SIGWINCH, originalSignalHandler) diff --git a/.templates/pihole/service.yml b/.templates/pihole/service.yml index 69e84fe9c..970fded45 100644 --- a/.templates/pihole/service.yml +++ b/.templates/pihole/service.yml @@ -8,7 +8,7 @@ pihole: - "67:67/udp" environment: - TZ=${TZ:-Etc/UTC} - - WEBPASSWORD= + - WEBPASSWORD=${PIHOLE_ADMIN_PASSWORD:-} # see https://sensorsiot.github.io/IOTstack/Containers/Pi-hole/#adminPassword - INTERFACE=eth0 - FTLCONF_MAXDBDAYS=365 diff --git a/.templates/plex/build.py b/.templates/plex/build.py index 1ea53f3a8..8cfafc23d 100755 --- a/.templates/plex/build.py +++ b/.templates/plex/build.py @@ -1,14 +1,13 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -16,43 +15,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -74,17 +36,45 @@ def preBuild(): def checkForIssues(): return True + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass - -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'plex': - main() -else: - print("Error. '{}' Tried to run 'plex' config".format(currentServiceName)) + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + + +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/portainer-ce/build.py b/.templates/portainer-ce/build.py index 475562973..a14e46e4e 100755 --- a/.templates/portainer-ce/build.py +++ b/.templates/portainer-ce/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -22,7 +22,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -38,43 +37,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -165,13 +127,18 @@ def createMenu(): portainerCeBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -254,6 +221,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, portainerCeBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -300,17 +268,50 @@ def menuEntryPoint(): #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'portainer-ce': - main() -else: - print("Error. '{}' Tried to run 'portainer-ce' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/python-matter-server/build.py b/.templates/python-matter-server/build.py index 261aceb8b..386a592ad 100755 --- a/.templates/python-matter-server/build.py +++ b/.templates/python-matter-server/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -22,7 +22,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -46,43 +45,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -258,13 +220,18 @@ def createMenu(): matterBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -356,6 +323,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, matterBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -398,17 +366,50 @@ def menuEntryPoint(): return True + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'python-matter-server': - main() -else: - print("Error. '{}' Tried to run 'python-matter-server' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/python-matter-server/select_extras.py b/.templates/python-matter-server/select_extras.py index c07d15c9b..ff0f7de62 100755 --- a/.templates/python-matter-server/select_extras.py +++ b/.templates/python-matter-server/select_extras.py @@ -90,7 +90,7 @@ def renderHotZone(term, renderType, menu, selection, hotzoneLocation, paddingBef global paginationSize selectedTextLength = len("-> ") - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") if paginationStartIndex >= 1: print(term.center("{b} {uaf} {uaf}{uaf}{uaf} {ual} {b}".format( @@ -273,6 +273,7 @@ def saveAddonList(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, mainMenuList, currentMenuItemIndex) + needsRender = 0 dockerCommandsSelectionInProgress = True with term.cbreak(): while dockerCommandsSelectionInProgress: @@ -325,6 +326,6 @@ def saveAddonList(): return True -originalSignalHandler = signal.getsignal(signal.SIGINT) +originalSignalHandler = signal.getsignal(signal.SIGWINCH) main() signal.signal(signal.SIGWINCH, originalSignalHandler) diff --git a/.templates/python/build.py b/.templates/python/build.py index 9284091a0..960e7fa4e 100755 --- a/.templates/python/build.py +++ b/.templates/python/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -17,7 +17,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -36,43 +35,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -123,17 +85,45 @@ def checkEnvFiles(): # End Supporting functions # ##################################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass - -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'python': - main() -else: - print("Error. '{}' Tried to run 'python' config".format(currentServiceName)) + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + + +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/rtl_433/build.py b/.templates/rtl_433/build.py index 70cc4ee75..e1396b785 100755 --- a/.templates/rtl_433/build.py +++ b/.templates/rtl_433/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -16,7 +16,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -34,43 +33,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -110,17 +72,45 @@ def checkForIssues(): # End Supporting functions # ##################################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass - -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'rtl_433': - main() -else: - print("Error. '{}' Tried to run 'rtl_433' config".format(currentServiceName)) + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + + +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/transmission/build.py b/.templates/transmission/build.py index 74c6d63a6..ed3203f8c 100755 --- a/.templates/transmission/build.py +++ b/.templates/transmission/build.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly def main(): import os + import signal import time from blessed import Terminal from deps.chars import specialChars, commonTopBorder, commonBottomBorder, commonEmptyLine, padText @@ -15,7 +16,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -33,43 +33,6 @@ def main(): documentationHint = 'https://sensorsiot.github.io/IOTstack/' - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -78,37 +41,20 @@ def runChecks(): # This function is optional, and will run after the docker-compose.yml file is written to disk. def postBuild(): - if not os.path.exists(serviceVolume): - try: - os.makedirs(serviceVolume, exist_ok=True) - print("Created", serviceVolume, "for", currentServiceName) - except Exception as err: - print("Error creating directory", currentServiceName) - print(err) - if not os.path.exists(serviceVolume + "/downloads"): - try: - os.mkdir(serviceVolume + "/downloads") - print("Created", serviceVolume + "/downloads", "for", currentServiceName) - except Exception as err: - print("Error creating downloads directory", currentServiceName) - print(err) - - if not os.path.exists(serviceVolume + "/watch"): - try: - os.makedirs(serviceVolume + "/watch", exist_ok=True) - print("Created", serviceVolume + "/watch", "for", currentServiceName) - except Exception as err: - print("Error creating watch directory", currentServiceName) - print(err) - - if not os.path.exists(serviceVolume + "/config"): - try: - os.makedirs(serviceVolume + "/config", exist_ok=True) - print("Created", serviceVolume + "/config", "for", currentServiceName) - except Exception as err: - print("Error creating config directory", currentServiceName) - print(err) - + requiredDirectories = [ + serviceVolume, + serviceVolume + "/downloads", + serviceVolume + "/watch", + serviceVolume + "/config", + ] + try: + for directory in requiredDirectories: + if not os.path.exists(directory): + os.makedirs(directory, exist_ok=True) + print("Created", directory, "for", currentServiceName) + except OSError as err: + print("Error creating Transmission directories: %s" % err) + return False return True # This function is optional, and will run just before the build docker-compose.yml code. @@ -187,13 +133,18 @@ def createMenu(): transmissionBuildOptions.append(["Go back", goBack]) def runOptionsMenu(): - createMenu() - menuEntryPoint() - return True + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + createMenu() + menuEntryPoint() + return True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -276,6 +227,7 @@ def menuEntryPoint(): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, transmissionBuildOptions, currentMenuItemIndex) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: @@ -321,17 +273,50 @@ def menuEntryPoint(): # End menu section #################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def runOptionsMenu(context): + """Open this service's interactive configuration menu.""" + return _runHook(context, "runOptionsMenu") + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'transmission': - main() -else: - print("Error. '{}' Tried to run 'transmission' config".format(currentServiceName)) +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/webthingsio_gateway/build.py b/.templates/webthingsio_gateway/build.py index 0be4ae8c8..789bb7e69 100755 --- a/.templates/webthingsio_gateway/build.py +++ b/.templates/webthingsio_gateway/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -16,7 +16,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -35,43 +34,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -122,17 +84,45 @@ def checkEnvFiles(): # End Supporting functions # ##################################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass - -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'webthingsio_gateway': - main() -else: - print("Error. '{}' Tried to run 'webthingsio_gateway' config".format(currentServiceName)) + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + + +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/zigbee2mqtt/build.py b/.templates/zigbee2mqtt/build.py index 729a3bd7a..bff8dbf37 100755 --- a/.templates/zigbee2mqtt/build.py +++ b/.templates/zigbee2mqtt/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -15,7 +15,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -33,43 +32,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -93,17 +55,45 @@ def preBuild(): # End Supporting functions # ##################################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass - -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'zigbee2mqtt': - main() -else: - print("Error. '{}' Tried to run 'zigbee2mqtt' config".format(currentServiceName)) + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + + +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/.templates/zigbee2mqtt_assistant/build.py b/.templates/zigbee2mqtt_assistant/build.py index b05ddf7e1..8f5d5b79d 100755 --- a/.templates/zigbee2mqtt_assistant/build.py +++ b/.templates/zigbee2mqtt_assistant/build.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 + issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks haltOnErrors = True # Main wrapper function. Required to make local vars work correctly @@ -15,7 +15,6 @@ def main(): global dockerComposeServicesYaml # The loaded memory YAML of all checked services global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result global currentServiceName # Name of the current service global issues # Returned issues dict global haltOnErrors # Turn on to allow erroring @@ -33,43 +32,6 @@ def main(): # runtime vars portConflicts = [] - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - # This service will not check anything unless this is set # This function is optional, and will run each time the menu is rendered def runChecks(): @@ -100,17 +62,45 @@ def checkForIssues(): # End Supporting functions # ##################################### + hook = locals().get(toRun) + if hook is None: + raise ValueError("Unknown service hook '%s'" % toRun) if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass - -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'zigbee2mqtt_assistant': - main() -else: - print("Error. '{}' Tried to run 'zigbee2mqtt_assistant' config".format(currentServiceName)) + return hook() + try: + return hook() + except Exception: + return None + +def _runHook(context, action): + """Adapt the service's established implementation to hook API v2.""" + global dockerComposeServicesYaml + global currentServiceName + global renderMode + global toRun + + dockerComposeServicesYaml = context.services + currentServiceName = context.serviceName + renderMode = context.renderMode + toRun = action + result = main() + context.services = dockerComposeServicesYaml + return result + + +def runChecks(context): + """Return build issues for the currently selected Compose services.""" + global issues + issues = {} + _runHook(context, "runChecks") + return issues + + +def preBuild(context): + """Run this service's pre-build work.""" + return _runHook(context, "preBuild") + + +def postBuild(context): + """Run this service's post-build work.""" + return _runHook(context, "postBuild") diff --git a/docs/Basic_setup/Backup-and-Restore.md b/docs/Basic_setup/Backup-and-Restore.md index 48de688a1..1d1fdbc77 100644 --- a/docs/Basic_setup/Backup-and-Restore.md +++ b/docs/Basic_setup/Backup-and-Restore.md @@ -38,6 +38,8 @@ Backups: * You can find the backups in the ./backups/ folder. With rolling being in ./backups/rolling/ and date backups in ./backups/backup/ * Log files can also be found in the ./backups/logs/ directory. + * Full backups include the root `.env` file so required credentials and device paths can be restored. Treat backup archives as sensitive data. + ### Examples: @@ -71,7 +73,7 @@ There are 2 ways to run a restore: The restore script takes 2 arguments: * Filename: The name of the backup file. The file must be present in the `./backups/` directory, or a subfolder in it. That means it should be moved from `./backups/backup` to `./backups/`, or that you need to specify the `backup` portion of the directory (see examples) -* NoAsk: If a second parameter is present, is acts as setting the no ask flag to true. +* NoAsk: Pass the literal `noask` as the second parameter to skip the destructive-operation confirmation prompt. ## Pre and post script hooks The script checks if there are any pre and post back up hooks to execute commands. Both of these files will be included in the backup, and have also been added to the `.gitignore` file, so that they will not be touched when IOTstack updates. diff --git a/docs/Basic_setup/Default-Configs.md b/docs/Basic_setup/Default-Configs.md index 5280eb4e7..390e1eeb7 100644 --- a/docs/Basic_setup/Default-Configs.md +++ b/docs/Basic_setup/Default-Configs.md @@ -2,7 +2,7 @@ Here you can find a list of the default mode and ports used by each service found in the .templates directory. -This list can be generated by running the default_ports_md_generator.sh script. +This list can be generated by running `python3 scripts/default_ports_md_generator.py`. | Service Name | Mode | Port(s)
*External:Internal* | | ------------ | -----| --------------- | diff --git a/docs/Basic_setup/Proxmox.md b/docs/Basic_setup/Proxmox.md new file mode 100644 index 000000000..9e66dfe4a --- /dev/null +++ b/docs/Basic_setup/Proxmox.md @@ -0,0 +1,51 @@ +# Proxmox virtual machine + +IOTstack should run in a Proxmox virtual machine rather than an LXC container. A virtual machine avoids complications with Docker nesting and makes hardware passthrough easier to manage. + +## Recommended virtual machine settings + +Use the following settings when creating the virtual machine: + +| Setting | Recommended value | +| --- | --- | +| Name | `iotstack` | +| Installation media | Debian 12 (Bookworm) 64-bit netinst ISO | +| Guest OS type | Linux, kernel 6.x | +| Machine | `q35` | +| BIOS | OVMF (UEFI), with an EFI disk | +| QEMU Guest Agent | Enabled | +| TPM | None | +| Disk controller | VirtIO SCSI Single | +| Disk | 64 GB minimum; 100 GB recommended | +| CPU type | `host` | +| CPU allocation | 2 cores minimum; 4 cores recommended | +| Memory | 4 GB minimum; 8 GB recommended | +| Network bridge | `vmbr0` | +| Network device | VirtIO | + +For the virtual disk, enable discard and IO thread. Enable SSD emulation when the underlying Proxmox storage is SSD-backed. Disabling memory ballooning gives Docker containers and databases a predictable amount of memory. + +Give the guest a stable address using either a DHCP reservation or static network configuration. Enable **Start at boot** after confirming that the installation works correctly. + +## Debian installation + +Create a normal, non-root user during installation. IOTstack and many of its containers expect the first user to have UID 1000. Select **SSH server** and **standard system utilities**; a desktop environment is not required. Guided partitioning with one ext4 filesystem is sufficient for a typical installation. + +After Debian starts, update it and install the Proxmox guest agent: + +``` console +$ sudo apt update +$ sudo apt full-upgrade -y +$ sudo apt install -y curl qemu-guest-agent +$ sudo systemctl enable --now qemu-guest-agent +``` + +Run the IOTstack installer as the normal user, without `sudo`: + +``` console +$ curl -fsSL https://raw.githubusercontent.com/SensorsIot/IOTstack/master/install.sh | bash +``` + +## USB devices + +Services that use a physical USB device require that device to be passed through from Proxmox to the virtual machine. Configure USB passthrough before attempting to use such a service. Device-specific passthrough instructions are outside the scope of this guide. diff --git a/docs/Basic_setup/index.md b/docs/Basic_setup/index.md index e3a338f06..a5ed71cf0 100644 --- a/docs/Basic_setup/index.md +++ b/docs/Basic_setup/index.md @@ -32,6 +32,8 @@ IOTstack makes the following assumptions: - an Intel-based Mac running macOS plus Parallels with a Debian guest. - an Intel-based platform running Proxmox with a Debian guest. + See [Proxmox virtual machine](Proxmox.md) for recommended guest settings. + 2. Your host or guest system is running a reasonably-recent version of Debian or an operating system which is downstream of Debian in the Linux family tree, such as Raspberry Pi OS (aka "Raspbian") or Ubuntu. IOTstack is known to work in 32-bit mode but not all containers have images on DockerHub that support 320bit mode. If you are setting up a new system from scratch, you should choose a 64-bit option. diff --git a/docs/Containers/MJPEG-Streamer.md b/docs/Containers/MJPEG-Streamer.md index ecaba0ca7..0a7f520c2 100644 --- a/docs/Containers/MJPEG-Streamer.md +++ b/docs/Containers/MJPEG-Streamer.md @@ -99,8 +99,8 @@ If you don't get a sensible response to the `ls` command then try disconnecting variable | default | remark ---------------------------------|:-------------:|------------------------------ -`MJPG_STREAMER_USERNAME` | container ID | *changes each time the container is recreated* -`MJPG_STREAMER_PASSWORD` | random UUID | *changes each time the container restarts* +`MJPG_STREAMER_USERNAME` | `iotstack` | may be overridden in `.env` +`MJPG_STREAMER_PASSWORD` | `IOtSt4ckMJPG` | may be kept or changed in the build menu `MJPG_STREAMER_SIZE` | `640x480` | should be one of your camera's natural resolutions `MJPG_STREAMER_FPS` | `5` | frames per second @@ -120,7 +120,7 @@ To initialise your environment, begin by using a text editor (eg `vim`, `nano`) TZ=Australia/Sydney ``` -2. The access credentials default to random values which change each time the container starts. This is reasonably secure but is unlikely to be useful in practice, so you need to invent some credentials of your own. Example: +2. The access credentials have stable defaults so you cannot lose a randomly-generated password. Use the service's Password Options menu to keep the default, enter your own, or generate and save a random password. You can also override either credential in `.env`. Example: ``` MJPG_STREAMER_USERNAME=streamer diff --git a/docs/Developers/BuildStack-RandomPassword.md b/docs/Developers/BuildStack-RandomPassword.md index e25fd91af..42010901f 100644 --- a/docs/Developers/BuildStack-RandomPassword.md +++ b/docs/Developers/BuildStack-RandomPassword.md @@ -1,489 +1,72 @@ -# Build Stack Random Services Password +# Build Stack Password Options -This page explains how to have a service generate a random password during build time. This will require that your service have a working options menu. +IOTstack creates password controls automatically from Compose environment variables. A service author only needs to describe the variable in `service.yml`; do not add password code to `build.py` or create a `passwords.py` file. -Keep in mind that updating strings in a service's yaml config isn't limited to passwords. +## Required passwords -## A word of caution -Many services often set a password on their initial spin up and store it internally. That means if if the password is changed by the menu afterwards, it may not be reflected in the service. By default the password specified in the documentation should be used, unless the user specifically selected to use a randomly generated one. In the future, the feature to specify a password manually may be added in, much like how ports can be customised. +Use Compose required-variable interpolation when the stack must not build without a value: -## A basic example -Inside the service's `service.yml` file, a special string can be added in for the build script to find and replace. Commonly the string is `%randomPassword%`, but technically any string can be used. The same string can be used multiple times for the same password to be used multiple times, and/or multiple difference strings can be used for multiple passwords. ``` yaml - mariadb: - image: linuxserver/mariadb - container_name: mariadb - environment: - - MYSQL_ROOT_PASSWORD=%randomAdminPassword% - - MYSQL_DATABASE=default - - MYSQL_USER=mariadbuser - - MYSQL_PASSWORD=%randomPassword% -``` - -These strings will be updated during the Prebuild Hook stage when building. The code to make this happen is shown below. +gitea: + environment: + - GITEA__database__PASSWD=${GITEA_DB_PASSWORD:?eg echo GITEA_DB_PASSWORD=userPassword >>~/IOTstack/.env} -## Code commonly used to update passwords -This code can basically be copy-pasted into your service's `build.py` file. You are welcome to expand upon it if required. It will probably be refactored into a utils function in the future to adear to DRY (Don't Repeat Yourself) practices. +gitea_db: + environment: + - MYSQL_PASSWORD=${GITEA_DB_PASSWORD:?eg echo GITEA_DB_PASSWORD=userPassword >>~/IOTstack/.env} ``` -def preBuild(): - # Multi-service load. Most services only include a single service. The exception being NextCloud where the database information needs to match between NextCloud and MariaDB (as defined in NextCloud's 'service.yml' file, not IOTstack's MariaDB). - with open((r'%s/' % serviceTemplate) + servicesFileName) as objServiceFile: - serviceYamlTemplate = yaml.load(objServiceFile) - - oldBuildCache = {} - try: - with open(r'%s' % buildCache) as objBuildCache: # Load previous build, if it exists - oldBuildCache = yaml.load(objBuildCache) - except: - pass - - buildCacheServices = {} - if "services" in oldBuildCache: # If a previous build does exist, load it so that we can reuse the password from it if required. - buildCacheServices = oldBuildCache["services"] - if not os.path.exists(serviceService): # Create the service directory for the service - os.makedirs(serviceService, exist_ok=True) +The build menu will: - # Check if buildSettings file exists (from previous build), or create one if it doesn't (in the else block). - if os.path.exists(buildSettings): - # Password randomisation - with open(r'%s' % buildSettings) as objBuildSettingsFile: - piHoleYamlBuildOptions = yaml.load(objBuildSettingsFile) - if ( - piHoleYamlBuildOptions["databasePasswordOption"] == "Randomise database password for this build" - or piHoleYamlBuildOptions["databasePasswordOption"] == "Randomise database password every build" - or deconzYamlBuildOptions["databasePasswordOption"] == "Use default password for this build" - ): - - if deconzYamlBuildOptions["databasePasswordOption"] == "Use default password for this build": - newAdminPassword = "######" # Update to what's specified in your documentation - newPassword = "######" # Update to what's specified in your documentation - else: - # Generate our passwords - newAdminPassword = generateRandomString() - newPassword = generateRandomString() - - # Here we loop through each service included in the current service's `service.yml` file and update the password strings. - for (index, serviceName) in enumerate(serviceYamlTemplate): - dockerComposeServicesYaml[serviceName] = serviceYamlTemplate[serviceName] - if "environment" in serviceYamlTemplate[serviceName]: - for (envIndex, envName) in enumerate(serviceYamlTemplate[serviceName]["environment"]): - envName = envName.replace("%randomPassword%", newPassword) - envName = envName.replace("%randomAdminPassword%", newAdminPassword) - dockerComposeServicesYaml[serviceName]["environment"][envIndex] = envName +- report a short build issue while `GITEA_DB_PASSWORD` is missing; +- add a **Password options** submenu to Gitea; +- offer the documented default (`userPassword`), a custom password, or a generated password; and +- save the selected value in `.env`, where it can be viewed again later. - # If the user had selected to only update the password once, ensure the build options file is updated. - if (piHoleYamlBuildOptions["databasePasswordOption"] == "Randomise database password for this build"): - piHoleYamlBuildOptions["databasePasswordOption"] = "Do nothing" - with open(buildSettings, 'w') as outputFile: - yaml.dump(piHoleYamlBuildOptions, outputFile) - else: # Do nothing - don't change password - for (index, serviceName) in enumerate(buildCacheServices): - if serviceName in buildCacheServices: # Load service from cache if exists (to maintain password) - dockerComposeServicesYaml[serviceName] = buildCacheServices[serviceName] - else: - dockerComposeServicesYaml[serviceName] = serviceYamlTemplate[serviceName] +The text after `:?` should contain `VARIABLE=defaultValue`. This supplies the default shown by the menu while Compose still requires the user to make an explicit choice. - # Build options file didn't exist, so create one, and also use default password (default action). - else: - print("PiHole Warning: Build settings file not found, using default password") - time.sleep(1) - newAdminPassword = "######" # Update to what's specified in your documentation - newPassword = "######" # Update to what's specified in your documentation - for (index, serviceName) in enumerate(serviceYamlTemplate): - dockerComposeServicesYaml[serviceName] = serviceYamlTemplate[serviceName] - if "environment" in serviceYamlTemplate[serviceName]: - for (envIndex, envName) in enumerate(serviceYamlTemplate[serviceName]["environment"]): - envName = envName.replace("%randomPassword%", newPassword) - envName = envName.replace("%randomAdminPassword%", newAdminPassword) - dockerComposeServicesYaml[serviceName]["environment"][envIndex] = envName - piHoleYamlBuildOptions = { - "version": "1", - "application": "IOTstack", - "service": "PiHole", - "comment": "PiHole Build Options" - } +## Optional passwords - piHoleYamlBuildOptions["databasePasswordOption"] = "Do nothing" - with open(buildSettings, 'w') as outputFile: - yaml.dump(piHoleYamlBuildOptions, outputFile) +Use Compose default-value interpolation if the service may start without an explicit `.env` entry: - return True +``` yaml +deconz: + environment: + - DECONZ_VNC_PASSWORD=${DECONZ_VNC_PASSWORD:-IOtSt4ckDec0nZ} ``` -## Code for your service's menu -While not needed, since the default action is to create a random password, it is a good idea to allow the user to choose what to do. This can be achieved by giving them access to a password menu. This code can be placed in your service's `build.py` file, that will show a new menu option, allowing users to select it and be taken to a password settings screen. +The password submenu is still created. Choosing **Use default** writes that default to `.env`; choosing **Generate** creates one cryptographically random value and saves it. Generated passwords are not regenerated during later builds. -Remember that you need to have an already working menu, and to place this code into it. +An empty default is valid for services where it intentionally means "no password": +``` yaml +pihole6: + environment: + FTLCONF_webserver_api_password: ${PIHOLE_ADMIN_PASSWORD:-} ``` -import signal - -... - -def setPasswordOptions(): - global needsRender - global hasRebuiltAddons - passwordOptionsMenuFilePath = "./.templates/{currentService}/passwords.py".format(currentService=currentServiceName) - with open(passwordOptionsMenuFilePath, "rb") as pythonDynamicImportFile: - code = compile(pythonDynamicImportFile.read(), passwordOptionsMenuFilePath, "exec") - execGlobals = { - "currentServiceName": currentServiceName, - "renderMode": renderMode - } - execLocals = {} - screenActive = False - exec(code, execGlobals, execLocals) - signal.signal(signal.SIGWINCH, onResize) - screenActive = True - needsRender = 1 -... +The menu labels this choice as **no password** so the result is explicit. -def createMenu(): - global yourServicesBuildOptions - global serviceService +## Multiple services sharing one password - yourServicesBuildOptions = [] - yourServicesBuildOptions.append([ - "Your Service Password Options", - setPasswordOptions - ]) +Use the same variable name everywhere the credential must match. It will appear only once in the password submenu: - yourServicesBuildOptions.append(["Go back", goBack]) - -``` +``` yaml +nextcloud: + environment: + - MYSQL_PASSWORD=${NEXTCLOUD_DB_USER_PASSWORD:?eg echo NEXTCLOUD_DB_USER_PASSWORD=IOtSt4ckmySqlDbPw >>~/IOTstack/.env} -## Password settings screen -The code for the Password settings is lengthy, but it's pasted here for convienence +nextcloud_db: + environment: + - MYSQL_PASSWORD=${NEXTCLOUD_DB_USER_PASSWORD:?eg echo NEXTCLOUD_DB_USER_PASSWORD=IOtSt4ckmySqlDbPw >>~/IOTstack/.env} ``` -#!/usr/bin/env python3 - -import signal - -def main(): - from blessed import Terminal - from deps.chars import specialChars, commonTopBorder, commonBottomBorder, commonEmptyLine - from deps.consts import servicesDirectory, templatesDirectory, buildSettingsFileName - import time - import subprocess - import ruamel.yamls - import os - - global signal - global currentServiceName - global menuSelectionInProgress - global mainMenuList - global currentMenuItemIndex - global renderMode - global paginationSize - global paginationStartIndex - global hideHelpText - - yaml = ruamel.yaml.YAML() - yaml.preserve_quotes = True - - try: # If not already set, then set it. - hideHelpText = hideHelpText - except: - hideHelpText = False - - term = Terminal() - hotzoneLocation = [((term.height // 16) + 6), 0] - paginationToggle = [10, term.height - 25] - paginationStartIndex = 0 - paginationSize = paginationToggle[0] - - serviceService = servicesDirectory + currentServiceName - serviceTemplate = templatesDirectory + currentServiceName - buildSettings = serviceService + buildSettingsFileName - - def goBack(): - global menuSelectionInProgress - global needsRender - menuSelectionInProgress = False - needsRender = 1 - return True - - mainMenuList = [] - - hotzoneLocation = [((term.height // 16) + 6), 0] - - menuSelectionInProgress = True - currentMenuItemIndex = 0 - menuNavigateDirection = 0 - - # Render Modes: - # 0 = No render needed - # 1 = Full render - # 2 = Hotzone only - needsRender = 1 - - def onResize(sig, action): - global mainMenuList - global currentMenuItemIndex - mainRender(1, mainMenuList, currentMenuItemIndex) - - def generateLineText(text, textLength=None, paddingBefore=0, lineLength=64): - result = "" - for i in range(paddingBefore): - result += " " - - textPrintableCharactersLength = textLength - - if (textPrintableCharactersLength) == None: - textPrintableCharactersLength = len(text) - - result += text - remainingSpace = lineLength - textPrintableCharactersLength - - for i in range(remainingSpace): - result += " " - - return result - - def renderHotZone(term, renderType, menu, selection, hotzoneLocation, paddingBefore = 4): - global paginationSize - selectedTextLength = len("-> ") - - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) - - if paginationStartIndex >= 1: - print(term.center("{b} {uaf} {uaf}{uaf}{uaf} {ual} {b}".format( - b=specialChars[renderMode]["borderVertical"], - uaf=specialChars[renderMode]["upArrowFull"], - ual=specialChars[renderMode]["upArrowLine"] - ))) - else: - print(term.center(commonEmptyLine(renderMode))) - - for (index, menuItem) in enumerate(menu): # Menu loop - if index >= paginationStartIndex and index < paginationStartIndex + paginationSize: - lineText = generateLineText(menuItem[0], paddingBefore=paddingBefore) - - # Menu highlight logic - if index == selection: - formattedLineText = '-> {t.blue_on_green}{title}{t.normal} <-'.format(t=term, title=menuItem[0]) - paddedLineText = generateLineText(formattedLineText, textLength=len(menuItem[0]) + selectedTextLength, paddingBefore=paddingBefore - selectedTextLength) - toPrint = paddedLineText - else: - toPrint = '{title}{t.normal}'.format(t=term, title=lineText) - # ##### - - # Menu check render logic - if menuItem[1]["checked"]: - toPrint = " (X) " + toPrint - else: - toPrint = " ( ) " + toPrint - - toPrint = "{bv} {toPrint} {bv}".format(bv=specialChars[renderMode]["borderVertical"], toPrint=toPrint) # Generate border - toPrint = term.center(toPrint) # Center Text (All lines should have the same amount of printable characters) - # ##### - print(toPrint) - - if paginationStartIndex + paginationSize < len(menu): - print(term.center("{b} {daf} {daf}{daf}{daf} {dal} {b}".format( - b=specialChars[renderMode]["borderVertical"], - daf=specialChars[renderMode]["downArrowFull"], - dal=specialChars[renderMode]["downArrowLine"] - ))) - else: - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - - - def mainRender(needsRender, menu, selection): - global paginationStartIndex - global paginationSize - term = Terminal() - - if selection >= paginationStartIndex + paginationSize: - paginationStartIndex = selection - (paginationSize - 1) + 1 - needsRender = 1 - - if selection <= paginationStartIndex - 1: - paginationStartIndex = selection - needsRender = 1 - - if needsRender == 1: - print(term.clear()) - print(term.move_y(term.height // 16)) - print(term.black_on_cornsilk4(term.center('IOTstack YourServices Password Options'))) - print("") - print(term.center(commonTopBorder(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Select Password Option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - - if needsRender >= 1: - renderHotZone(term, needsRender, menu, selection, hotzoneLocation) - - if needsRender == 1: - print(term.center(commonEmptyLine(renderMode))) - if not hideHelpText: - if term.height < 32: - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Not enough vertical room to render controls help text {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - else: - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Controls: {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Space] to select option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Up] and [Down] to move selection cursor {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [H] Show/hide this text {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Enter] to build and save option {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Escape] to cancel changes {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonBottomBorder(renderMode))) - - def runSelection(selection): - import types - if len(mainMenuList[selection]) > 1 and isinstance(mainMenuList[selection][1], types.FunctionType): - mainMenuList[selection][1]() - else: - print(term.green_reverse('IOTstack Error: No function assigned to menu item: "{}"'.format(mainMenuList[selection][0]))) - - def isMenuItemSelectable(menu, index): - if len(menu) > index: - if len(menu[index]) > 1: - if "skip" in menu[index][1] and menu[index][1]["skip"] == True: - return False - return True - - def loadOptionsMenu(): - global mainMenuList - mainMenuList.append(["Use default password for this build", { "checked": True }]) - mainMenuList.append(["Randomise database password for this build", { "checked": False }]) - mainMenuList.append(["Randomise database password every build", { "checked": False }]) - mainMenuList.append(["Do nothing", { "checked": False }]) - - def checkMenuItem(selection): - global mainMenuList - for (index, menuItem) in enumerate(mainMenuList): - mainMenuList[index][1]["checked"] = False - - mainMenuList[selection][1]["checked"] = True - - def saveOptions(): - try: - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - - if os.path.exists(buildSettings): - with open(r'%s' % buildSettings) as objBuildSettingsFile: - yourServicesYamlBuildOptions = yaml.load(objBuildSettingsFile) - else: - yourServices = { - "version": "1", - "application": "IOTstack", - "service": "Your Service", - "comment": "Your Service Build Options" - } - - yourServices["databasePasswordOption"] = "" - - for (index, menuOption) in enumerate(mainMenuList): - if menuOption[1]["checked"]: - yourServices["databasePasswordOption"] = menuOption[0] - break - - with open(buildSettings, 'w') as outputFile: - yaml.dump(yourServices, outputFile) - - except Exception as err: - print("Error saving Your Services Password options", currentServiceName) - print(err) - return False - global hasRebuiltHardwareSelection - hasRebuiltHardwareSelection = True - return True - - def loadOptions(): - try: - if not os.path.exists(serviceService): - os.makedirs(serviceService, exist_ok=True) - - if os.path.exists(buildSettings): - with open(r'%s' % buildSettings) as objBuildSettingsFile: - yourServicesYamlBuildOptions = yaml.load(objBuildSettingsFile) - - for (index, menuOption) in enumerate(mainMenuList): - if menuOption[0] == yourServicesYamlBuildOptions["databasePasswordOption"]: - checkMenuItem(index) - break - - except Exception as err: - print("Error loading Your Services Password options", currentServiceName) - print(err) - return False - return True - - - if __name__ == 'builtins': - global signal - term = Terminal() - signal.signal(signal.SIGWINCH, onResize) - loadOptionsMenu() - loadOptions() - with term.fullscreen(): - menuNavigateDirection = 0 - mainRender(needsRender, mainMenuList, currentMenuItemIndex) - menuSelectionInProgress = True - with term.cbreak(): - while menuSelectionInProgress: - menuNavigateDirection = 0 - - if not needsRender == 0: # Only rerender when changed to prevent flickering - mainRender(needsRender, mainMenuList, currentMenuItemIndex) - needsRender = 0 - - key = term.inkey(esc_delay=0.05) - if key.is_sequence: - if key.name == 'KEY_TAB': - if paginationSize == paginationToggle[0]: - paginationSize = paginationToggle[1] - else: - paginationSize = paginationToggle[0] - mainRender(1, mainMenuList, currentMenuItemIndex) - if key.name == 'KEY_DOWN': - menuNavigateDirection += 1 - if key.name == 'KEY_UP': - menuNavigateDirection -= 1 - if key.name == 'KEY_ENTER': - if saveOptions(): - return True - else: - print("Something went wrong. Try saving the list again.") - if key.name == 'KEY_ESCAPE': - menuSelectionInProgress = False - return True - elif key: - if key == ' ': # Space pressed - checkMenuItem(currentMenuItemIndex) # Update checked list - needsRender = 2 - elif key == 'h': # H pressed - if hideHelpText: - hideHelpText = False - else: - hideHelpText = True - mainRender(1, mainMenuList, currentMenuItemIndex) - if menuNavigateDirection != 0: # If a direction was pressed, find next selectable item - currentMenuItemIndex += menuNavigateDirection - currentMenuItemIndex = currentMenuItemIndex % len(mainMenuList) - needsRender = 2 +## Other sensitive settings - while not isMenuItemSelectable(mainMenuList, currentMenuItemIndex): - currentMenuItemIndex += menuNavigateDirection - currentMenuItemIndex = currentMenuItemIndex % len(mainMenuList) - return True +Names containing `SECRET`, `TOKEN`, `AUTHORIZATION`, or `API_KEY` are also discovered automatically. They appear as protected service settings, while names containing `PASSWORD` or `PASSWD` are grouped into the Password options submenu. - return True +## Service behavior to check -originalSignalHandler = signal.getsignal(signal.SIGINT) -main() -signal.signal(signal.SIGWINCH, originalSignalHandler) +Some applications read credentials only while initializing an empty data directory. Changing `.env` later may not update the credential stored inside an existing database. Document that behavior for the service and warn users before they change an initialized password. -``` \ No newline at end of file +Do not replace marker strings during `preBuild`, reload password-bearing services from the build cache, or generate a new secret on every build. Those patterns can hide the actual credential and can overwrite a value the user just saved. diff --git a/docs/Developers/BuildStack-Services.md b/docs/Developers/BuildStack-Services.md index 28a02017d..43cab6c2f 100644 --- a/docs/Developers/BuildStack-Services.md +++ b/docs/Developers/BuildStack-Services.md @@ -1,14 +1,16 @@ -# Build Stack Services system +# Build Stack Services -This page explains how the build stack system works for developers. +This page explains how to add a service to the build stack. -## How to define a new service -A service only requires 2 files: -* `service.yml` - Contains data for docker-compose -* `build.py` - Contains logic that the menu system uses. +## Smallest possible service + +A service normally needs only one file: + +- `service.yml` contains the Docker Compose service definition. +- `build.py` is optional and is only needed for custom checks, an interactive service-specific menu, or build-time file preparation. + +Create a directory under `.templates`. Its name must match a root service key in `service.yml`: -### A basic service -Inside the `service.yml` is where the service data for docker-compose is housed, for example: ``` yaml adminer: container_name: adminer @@ -17,177 +19,122 @@ adminer: ports: - "9080:8080" ``` -It is important that the service name match the directory that it's in - that means that the `adminer` service must be placed into a folder called `adminer` inside the `./.templates` directory. +For example, this definition belongs in `.templates/adminer/service.yml`. + +The easiest starting point is to copy `.templates/example_template`, rename the directory and YAML file, then delete `build.py` if no hooks are needed. + +## Environment settings and passwords + +Compose interpolation automatically creates build issues and settings screens. No Python menu code is needed. -### Basic build code for service -At the very least, the `build.py` requires the following code: +A required value uses `:?`: + +``` yaml +environment: + - PASSWORD=${MY_SERVICE_PASSWORD:?eg echo MY_SERVICE_PASSWORD=ChangeMe >>~/IOTstack/.env} ``` -#!/usr/bin/env python3 -issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks -haltOnErrors = True - -# Main wrapper function. Required to make local vars work correctly -def main(): - global currentServiceName # Name of the current service - - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - - # Entrypoint for execution - if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass - -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'adminer': # Make sure you update this. - main() -else: - print("Error. '{}' Tried to run 'adminer' config".format(currentServiceName)) +An optional value with a default uses `:-`: + +``` yaml +environment: + - PASSWORD=${MY_SERVICE_PASSWORD:-ChangeMe} ``` -This code doesn't have any port conflicting checking or menu code in it, and just allows the service to be built as is. The best way to learn on extending the functionality of the service's build script is to look at the other services' build scripts. You can also check out the advanced sections on adding menus and checking for issues for services though for a deeper explanation of specific situations. -### Basic code for a service that uses bash -If Python isn't your thing, here's a code blob you can copy and paste. Just be sure to update the lines where the comments start with `---` +Names containing `PASSWORD` or `PASSWD` appear in the shared Password options submenu. The user can keep the documented default, enter a value, or generate and save a random password. Names containing `SECRET`, `TOKEN`, `AUTHORIZATION`, or `API_KEY` are also exposed as protected service settings. + +See [Build Stack Password Options](./BuildStack-RandomPassword.md) for complete examples. + +## Optional hook file + +A hook file is an ordinary Python module. Add only the functions the service needs: + +``` python +def runChecks(context): + return {} + +def runOptionsMenu(context): + return None + +def preBuild(context): + return None + +def postBuild(context): + return None ``` + +There is no registration dictionary, dynamic execution, injected globals, class, or decorator. IOTstack imports the module and discovers these function names. + +### Hook context + +Every function receives one context object: + +- `context.serviceName` is the selected template name. +- `context.services` is the in-memory Compose services mapping. +- `context.renderMode` is the terminal character mode. +- `context.terminal` is the active Blessed terminal. + +Hooks may update `context.services` in place. `runChecks(context)` must return a dictionary; return `{}` when there are no issues. + +Return `False` from `preBuild` or `postBuild` when required work fails. The build will stop instead of writing a misleading success result. + +## Calling a Bash helper + +A Python hook can run a Bash script without using shell interpolation: + +``` python #!/usr/bin/env python3 -issues = {} # Returned issues dict -buildHooks = {} # Options, and others hooks -haltOnErrors = True - -# Main wrapper function. Required to make local vars work correctly -def main(): - import subprocess - global dockerComposeServicesYaml # The loaded memory YAML of all checked services - global toRun # Switch for which function to run when executed - global buildHooks # Where to place the options menu result - global currentServiceName # Name of the current service - global issues # Returned issues dict - global haltOnErrors # Turn on to allow erroring - - from deps.consts import servicesDirectory, templatesDirectory, volumesDirectory, servicesFileName - - # runtime vars - serviceVolume = volumesDirectory + currentServiceName # Unused in example - serviceService = servicesDirectory + currentServiceName # Unused in example - serviceTemplate = templatesDirectory + currentServiceName - - # This lets the menu know whether to put " >> Options " or not - # This function is REQUIRED. - def checkForOptionsHook(): - try: - buildHooks["options"] = callable(runOptionsMenu) - except: - buildHooks["options"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPreBuildHook(): - try: - buildHooks["preBuildHook"] = callable(preBuild) - except: - buildHooks["preBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForPostBuildHook(): - try: - buildHooks["postBuildHook"] = callable(postBuild) - except: - buildHooks["postBuildHook"] = False - return buildHooks - return buildHooks - - # This function is REQUIRED. - def checkForRunChecksHook(): - try: - buildHooks["runChecksHook"] = callable(runChecks) - except: - buildHooks["runChecksHook"] = False - return buildHooks - return buildHooks - - # This service will not check anything unless this is set - # This function is optional, and will run each time the menu is rendered - def runChecks(): - checkForIssues() - return [] - - # This function is optional, and will run after the docker-compose.yml file is written to disk. - def postBuild(): - return True - - # This function is optional, and will run just before the build docker-compose.yml code. - def preBuild(): - execComm = "bash {currentServiceTemplate}/build.sh".format(currentServiceTemplate=serviceTemplate) # --- You may want to change this - print("[Wireguard]: ", execComm) # --- Ensure to update the service name with yours - subprocess.call(execComm, shell=True) # This is where the magic happens - return True - - # ##################################### - # Supporting functions below - # ##################################### - - def checkForIssues(): - return True - - if haltOnErrors: - eval(toRun)() - else: - try: - eval(toRun)() - except: - pass - -# This check isn't required, but placed here for debugging purposes -global currentServiceName # Name of the current service -if currentServiceName == 'wireguard': # --- Ensure to update the service name with yours - main() -else: - print("Error. '{}' Tried to run 'wireguard' config".format(currentServiceName)) # --- Ensure to update the service name with yours - -``` \ No newline at end of file +import os +import subprocess + +from deps.consts import templatesDirectory + +def runChecks(context): + return {} + +def preBuild(context): + scriptPath = os.path.join( + templatesDirectory, + context.serviceName, + "build.sh", + ) + result = subprocess.run(["bash", scriptPath]) + if result.returncode != 0: + print("%s build helper failed." % context.serviceName) + return False + return True + +def postBuild(context): + return True +``` + +Keep checks read-only. File creation, privileged commands, and other mutations belong in `preBuild` or `postBuild`, not `runChecks`. + +Do not generate hidden credentials in a hook. Declare them in `service.yml` so the shared settings menu saves them in `.env`. + +## Templates with companion services + +A template may define multiple Compose services. The directory name must still match one root key: + +``` yaml +myapp: + image: example/myapp + +myapp_db: + image: mariadb +``` + +Selecting `myapp` loads both services. Deselecting it removes both, and saved settings for both are restored the next time the build menu opens. + +## Validation + +Before submitting a pull request, run: + +``` console +python3 -m unittest discover -v +docker-compose config -q +``` + +All bundled `build.py` files are inspected by the regression tests, including hook availability and return contracts. diff --git a/docs/Developers/Menu-System.md b/docs/Developers/Menu-System.md index 3048ed70a..36244b9c6 100644 --- a/docs/Developers/Menu-System.md +++ b/docs/Developers/Menu-System.md @@ -8,6 +8,8 @@ Originally this script was written in bash. After a while it became obvious that ## Menu Structure Each screen of the menu is its own Python script. You can find most of these in the `./scripts` directory. When you select an item from the menu, and it changes screens, it actually dynamically loads and executes that Python script. It passes data as required by placing it into the global variable space so that both the child and the parent script can access it. +This injected-global design applies to the menu screens themselves. Service `build.py` hooks use the simpler importlib/context API described below. + ### Injecting and getting globals in a child script ``` @@ -24,7 +26,7 @@ print(globalKeyName) # Will print out 'newValue' ### Reading and writing global variables in a child script ``` -def someFunction: +def someFunction(): global globalKeyName print(globalKeyName) # Will print out 'globalKeyValue' globalKeyName = "newValue" @@ -44,7 +46,7 @@ Is actually where the execution path runs, all the code above it is just declare It was obvious early on that the menu system would be slow on lower end devices, such as the Raspberry Pi, especially if it were rending a 4k terminal screen from a desktop via SSH. To mitigate this issue, not all of the screen is redrawn when there is a change. A "Hotzone" as it's called in the code, is usually rerendered when there's a change (such as pressing up or down to change an item selection, but not when scrolling). Full screen redraws are expensive and are only used when required, for example, when scrolling the pagination, selecting or deselecting a service, expanding or collapsing the menu and so on. ### Environments and encoding -At the very beginning of the main menu screen (`./scripts/main_menu.py`) the function `checkRenderOptions()` is run to determine what characters can be displayed on the screen. It will try various character sets, and eventually default to ASCII if none of the fancier stuff can be rendered. This setting is passed into of the sub menus through the submenu's global variables so that they don't have to recheck when they load. +At the very beginning of the main menu screen (`./scripts/menu_main.py`) the function `checkRenderOptions()` is run to determine what characters can be displayed on the screen. It will try various character sets, and eventually default to ASCII if none of the fancier stuff can be rendered. This setting is passed into of the sub menus through the submenu's global variables so that they don't have to recheck when they load. ### Sub-Menus @@ -56,35 +58,71 @@ Path: `./scripts/buildstack_menu.py` ### Loading -1. Upon loading, the Build Stack menu will get a list of folders inside the `./templates` directory and check for a `build.py` file inside each of them. This can be seen in the `generateTemplateList()` function, which is executed before the first rendering happens. -2. The menu will then check if the file `./services/docker-compose.save.yml` exists. This file is used to save the configuration of the last build. This happens in the `loadCurrentConfigs()` function. It is important that the service name in the compose file matches the folder name, any service that doesn't will either cause an error, or won't be loaded into the menu. -3. If a previous build did exist the menu will then run the `prepareMenuState()` function that basically checks which items should be ticked, and check for any issues with the ticked items by running `checkForIssues()`. +1. The Build Stack menu lists folders in `./.templates` that contain a `service.yml` file. A `build.py` file is optional. +2. The menu loads `./services/docker-compose.save.yml`, if present, to restore the previous selection and settings. +3. The menu prepares the selected state and runs checks for the restored services. + +### Adding a service hook + +The intended workflow is to copy `./.templates/example_template`, rename the directory, rename `example_service.yml` to `service.yml`, make its root service key match the new directory name, and optionally edit `build.py`. Service hooks are ordinary Python modules loaded with the standard-library `importlib` machinery. + +A new `build.py` may define any of these optional functions. Delete the functions the service does not need: + +```python +def runChecks(context): + return {} + +def runOptionsMenu(context): + pass + +def preBuild(context): + pass + +def postBuild(context): + pass +``` + +No classes, decorators, registration dictionaries, package installation, or global declarations are required. IOTstack discovers the functions by name. Each function receives a context with four attributes: + +* `context.serviceName` - the service directory/name currently being processed. +* `context.services` - the in-memory Compose services mapping. Hooks may update it in place. +* `context.renderMode` - the selected terminal character mode. +* `context.terminal` - the active blessed terminal, for an options UI. + +`runChecks(context)` must return a dictionary. Return `{}` when the service passes its checks. The other hooks may return `None`. + +The loader and validation live in `scripts/deps/service_hooks.py`. Contributors should not need to modify that file. + +Hook compatibility is defined by the callable names and context contract; no version declaration is required. ### Selection and deselection -When an item is selected, 3 things happen: -1. Update the UI variable (`menu`) with function `checkMenuItem(selectionIndex)` to let the user know the current state. -2. Update the array holding every checked item `setCheckedMenuItems()`. It uses the UI variable (`menu`) to know which items are set. -3. Check for any issues with the new list of selected items by running `checkForIssues()`. -### Check for options (submenus of services) -During a full render sequence (this is not a hotzone render), the build stack menu checks to see if each of the services has an options menu. It does this by executing the `build.py` script of each of the services and passing in `checkForOptionsHook` into the `toRun` global variable property to see if the script has a `runOptionsMenu` function. If the service's function result is true, without error, then the options text will appear up for that menu item. +When an item is selected, the menu updates its checked state, loads every Compose service from that template, and runs checks against the new selection. Deselecting it removes every Compose service owned by the template. + +### Check for options + +During a full render, the build menu checks the selected template for Compose environment settings and loads its optional `build.py` module. The options indicator is shown when automatic settings or a callable `runOptionsMenu(context)` function are available. ### Check for issues -When a service is selected or deselected on the menu, the `checkForIssues()` function is run. This function iterates through each of the selected menu items' folders executing the `build.py` script and passing in `checkForRunChecksHook` into the `toRun` global variable property to see if the script has a `runChecks` function. The `runChecks` function is different depending on the service, since each service has its own requirements. Generally though, the `runChecks` function should check for conflicting port conflicts again any of the other services that are enabled. The menu will still allow you to build the stack, even if issues are present, assumine there's no errors raised during the build process. + +When a service is selected or deselected, the menu calls its optional `runChecks(context)` function. Checks commonly detect port conflicts, missing dependent services, or required configuration files. The returned dictionary is displayed in the build issues panel. ### Prebuild hook -Pressing enter on the Build Stack menu kicks off the build process. The Build Stack menu will execute the `runPrebuildHook()` function. This function iterates through each of the selected menu items' folders executing the `build.py` script and passing in `checkForPreBuildHook` into the `toRun` global variable property to see if the script has a `preBuild` function. The `preBuild` function is different depending on the service, since each service has its own requirements. Some services may not even use the prebuild hook. The prebuild is very useful for setting up the services' configuration however. For example, it can be used to autogenerate a password for a paticular service, or copy and modify a configuration file from the `./.templates` directory into the `./services` or `./volumes` directory. + +After required environment settings pass validation, the build calls each selected service's optional `preBuild(context)` function. It can create configuration files or update `context.services` before Compose output is written. Credentials should be declared in `service.yml` and saved through the shared settings menu. ### Postbuild hook -The Build Stack menu will execute the `runPostBuildHook()` function in the final step of the build process, after the `docker-compose.yml` file has been written to disk. This function iterates through each of the selected menu items' folders executing the `build.py` script and passing in `checkForPostBuildHook` into the `toRun` global variable property to see if the script has a `postBuild` function. The `postBuild` function is different depending on the service, since each service has its own requirements. Most services won't require this function, but it can be useful for cleaning up temporary files and so on. + +After `docker-compose.yml` has been written, the menu calls each selected service's optional `postBuild(context)` function. Most services do not need one, but it can apply permissions or clean up temporary files. ### The build process The selected services' yaml configuration is already loaded into memory before the build stack process is started. -1. Run prebuildHooks. -2. Read `./.templates/docker-compose-base.yml` file into a in memory yaml structure. -3. Add selected services into the in memory structure. -4. If it exists merge the `./compose-override.yml` file into memory -5. Write the in memory yaml structure to disk `./docker-compose.yml`. -6. Run postbuildHooks. -7. Run `postbuild.sh` if it exists, with the list of services built. +1. Validate required Compose environment settings and stop without side effects when any are missing. +2. Run prebuild hooks. +3. Read `./.templates/docker-compose-base.yml` into an in-memory YAML structure. +4. Add selected services to the in-memory structure. +5. Merge `./compose-override.yml` when it exists. +6. Write the in-memory YAML structure to `./docker-compose.yml`. +7. Run postbuild hooks. +8. Run `postbuild.sh`, when present, with the list of services built. diff --git a/docs/Developers/index.md b/docs/Developers/index.md index ecf61d957..33c251b51 100644 --- a/docs/Developers/index.md +++ b/docs/Developers/index.md @@ -40,7 +40,7 @@ Services will grow over time, we may split up the buildstack menu into subsectio * `build.py` file is correct * Service allows for changing external WUI port from Build Stack's options menu if service uses a HTTP/S port * Use a default password, or allow the user to generate a random password for the service for initial installation. If the service asks to setup an account this can be ignored. -* Ensure [Default Configs](../Basic_setup/Default-Configs.md) is updated as required. A helper script (default_ports_md_generator.sh) exists to simplify this. +* Ensure [Default Configs](../Basic_setup/Default-Configs.md) is updated as required. Run `python3 scripts/default_ports_md_generator.py` to regenerate the table. * Must detect port conflicts with other services on [BuildStack](Menu-System.md) Menu. * `Pre` and `Post` hooks work with no errors. * Does not require user to edit config files in order to get the service running. diff --git a/menu.sh b/menu.sh index 94d867428..8e70be7aa 100755 --- a/menu.sh +++ b/menu.sh @@ -62,34 +62,22 @@ function minimum_version_check() { return 1 fi - if [ "${CURR_VERSION_MAJOR}" -ge $REQ_MIN_VERSION_MAJOR ]; then + if [ "${CURR_VERSION_MAJOR}" -gt "$REQ_MIN_VERSION_MAJOR" ]; then VERSION_GOOD="true" - echo "$VERSION_GOOD" - return 0 - else + elif [ "${CURR_VERSION_MAJOR}" -lt "$REQ_MIN_VERSION_MAJOR" ]; then VERSION_GOOD="false" - fi - - if [ "${CURR_VERSION_MAJOR}" -ge $REQ_MIN_VERSION_MAJOR ] && \ - [ "${CURR_VERSION_MINOR}" -ge $REQ_MIN_VERSION_MINOR ]; then + elif [ "${CURR_VERSION_MINOR}" -gt "$REQ_MIN_VERSION_MINOR" ]; then VERSION_GOOD="true" - echo "$VERSION_GOOD" - return 0 - else + elif [ "${CURR_VERSION_MINOR}" -lt "$REQ_MIN_VERSION_MINOR" ]; then VERSION_GOOD="false" - fi - - if [ "${CURR_VERSION_MAJOR}" -ge $REQ_MIN_VERSION_MAJOR ] && \ - [ "${CURR_VERSION_MINOR}" -ge $REQ_MIN_VERSION_MINOR ] && \ - [ "${CURR_VERSION_BUILD}" -ge $REQ_MIN_VERSION_BUILD ]; then + elif [ "${CURR_VERSION_BUILD}" -ge "$REQ_MIN_VERSION_BUILD" ]; then VERSION_GOOD="true" - echo "$VERSION_GOOD" - return 0 else VERSION_GOOD="false" fi echo "$VERSION_GOOD" + [ "$VERSION_GOOD" = "true" ] } function check_git_updates() diff --git a/scripts/backup.sh b/scripts/backup.sh index e22453f63..d8cb260b9 100755 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -25,7 +25,7 @@ # sudo bash ./scripts/backup.sh 2 pi # This will only produce a backup in the rollowing folder and change all the permissions to the 'pi' user. -if [ -d "./menu.sh" ]; then +if [ ! -f "./menu.sh" ]; then echo "./menu.sh file was not found. Ensure that you are running this from IOTstack's directory." exit 1 fi @@ -90,25 +90,39 @@ bash ./scripts/backup_restore/pre_backup_complete.sh >> $LOGFILE 2>&1 echo "./services/" >> $BACKUPLIST echo "./volumes/" >> $BACKUPLIST [ -f "./docker-compose.yml" ] && echo "./docker-compose.yml" >> $BACKUPLIST -[ -f "./docker-compose.override.yml" ] && echo "./docker-compose.yml" >> $BACKUPLIST +[ -f "./.env" ] && echo "./.env" >> $BACKUPLIST +[ -f "./docker-compose.override.yml" ] && echo "./docker-compose.override.yml" >> $BACKUPLIST [ -f "./compose-override.yml" ] && echo "./compose-override.yml" >> $BACKUPLIST -[ -f "./extra" ] && echo "./extra" >> $BACKUPLIST -[ -f "./.tmp/databases_backup" ] && echo "./.tmp/databases_backup" >> $BACKUPLIST +[ -e "./extra" ] && echo "./extra" >> $BACKUPLIST +[ -d "./.tmp/databases_backup" ] && echo "./.tmp/databases_backup" >> $BACKUPLIST [ -f "./postbuild.sh" ] && echo "./postbuild.sh" >> $BACKUPLIST [ -f "./post_backup.sh" ] && echo "./post_backup.sh" >> $BACKUPLIST [ -f "./pre_backup.sh" ] && echo "./pre_backup.sh" >> $BACKUPLIST +[ -f "./post_restore.sh" ] && echo "./post_restore.sh" >> $BACKUPLIST -sudo tar -czf $TMPBACKUPFILE -T $BACKUPLIST >> $LOGFILE 2>&1 +if ! sudo tar -czf "$TMPBACKUPFILE" -T "$BACKUPLIST" >> "$LOGFILE" 2>&1; then + echo "Backup archive creation failed." >> "$LOGFILE" + cat "$LOGFILE" + exit 2 +fi [ -f "$ROLLING" ] && ROLLINGOVERWRITTEN=1 && rm -rf $ROLLING sudo chown -R $USER:$USER $TMPDIR/backup* >> $LOGFILE 2>&1 if [[ "$BACKUPTYPE" -eq "1" || "$BACKUPTYPE" -eq "3" ]]; then - cp $TMPBACKUPFILE $BACKUPFILE + if ! cp "$TMPBACKUPFILE" "$BACKUPFILE"; then + echo "Failed to copy backup archive to '$BACKUPFILE'." >> "$LOGFILE" + cat "$LOGFILE" + exit 3 + fi fi if [[ "$BACKUPTYPE" -eq "2" || "$BACKUPTYPE" -eq "3" ]]; then - cp $TMPBACKUPFILE $ROLLING + if ! cp "$TMPBACKUPFILE" "$ROLLING"; then + echo "Failed to copy rolling archive to '$ROLLING'." >> "$LOGFILE" + cat "$LOGFILE" + exit 3 + fi fi if [[ "$BACKUPTYPE" -eq "2" || "$BACKUPTYPE" -eq "3" ]]; then diff --git a/scripts/backup_restore.py b/scripts/backup_restore.py index e13b8d0a5..7ac0f2f23 100755 --- a/scripts/backup_restore.py +++ b/scripts/backup_restore.py @@ -25,9 +25,12 @@ def main(): def runBackup(): global needsRender print("Execute Backup:") - subprocess.call("./scripts/backup.sh", shell=True) + returnCode = subprocess.call(["./scripts/backup.sh"]) print("") - print("Backup completed.") + if returnCode == 0: + print("Backup completed.") + else: + print("Backup failed with exit code %s." % returnCode) print("Press [Up] or [Down] arrow key to show the menu if it has scrolled too far.") time.sleep(1) needsRender = 1 @@ -71,9 +74,12 @@ def rCloneSetup(): def runRestore(): global needsRender print("Execute Restore:") - subprocess.call("./scripts/restore.sh", shell=True) + returnCode = subprocess.call(["./scripts/restore.sh"]) print("") - print("Restore completed.") + if returnCode == 0: + print("Restore process finished.") + else: + print("Restore failed with exit code %s." % returnCode) print("Press [Up] or [Down] arrow key to show the menu if it has scrolled too far.") time.sleep(1) needsRender = 1 @@ -114,7 +120,7 @@ def onResize(sig, action): def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -191,6 +197,7 @@ def isMenuItemSelectable(menu, index): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, mainMenuList, currentMenuItemIndex) + needsRender = 0 backupRestoreSelectionInProgress = True with term.cbreak(): while backupRestoreSelectionInProgress: @@ -235,6 +242,6 @@ def isMenuItemSelectable(menu, index): return True -originalSignalHandler = signal.getsignal(signal.SIGINT) +originalSignalHandler = signal.getsignal(signal.SIGWINCH) main() signal.signal(signal.SIGWINCH, originalSignalHandler) diff --git a/scripts/buildstack_menu.py b/scripts/buildstack_menu.py index b0c29108c..0e8db9374 100755 --- a/scripts/buildstack_menu.py +++ b/scripts/buildstack_menu.py @@ -6,15 +6,23 @@ def main(): import os - import time import ruamel.yaml - import math import sys import subprocess import traceback - from deps.chars import specialChars, commonTopBorder, commonBottomBorder, commonEmptyLine, padText - from deps.consts import servicesDirectory, templatesDirectory, volumesDirectory, buildCache, envFile, dockerPathOutput, servicesFileName, composeOverrideFile + from deps.chars import specialChars, commonTopBorder, commonBottomBorder, commonEmptyLine, commonTextLine, padText + from deps.consts import servicesDirectory, templatesDirectory, buildCache, envFile, dockerPathOutput, servicesFileName, composeOverrideFile from deps.yaml_merge import mergeYaml + from deps.service_templates import loadServiceTemplate, mergeServiceTemplate, removeServiceTemplate, restoreSavedServiceTemplates + from deps.menu_renderer import issuePanelHeight, pageSizeForTerminal, paginationStart, serviceOptionsMessage, terminalSupportsMenu + from deps.service_hooks import HookContext, serviceHookAvailable, runServiceHook + from deps.compose_environment import ( + requiredEnvironmentIssues, + configurableEnvironmentVariables, + restoreEnvironmentVariableReferences, + ) + from deps.environment_options import runEnvironmentOptions + from deps.issue_viewer import compactIssueRows, runIssueViewer from blessed import Terminal global signal global renderMode @@ -22,8 +30,8 @@ def main(): global paginationSize global paginationStartIndex global hideHelpText - global activeMenuLocation global lastSelection + global transientMessage yaml = ruamel.yaml.YAML() yaml.preserve_quotes = True @@ -38,21 +46,32 @@ def main(): templatesDirectoryFolders = next(os.walk(templatesDirectory))[1] term = Terminal() hotzoneLocation = [7, 0] # Top text - paginationToggle = [10, term.height - 22] # Top text + controls text paginationStartIndex = 0 - paginationSize = paginationToggle[0] - activeMenuLocation = 0 lastSelection = 0 + transientMessage = None try: # If not already set, then set it. hideHelpText = hideHelpText except: hideHelpText = False + reservedLines = 19 if hideHelpText else 27 + paginationSize = pageSizeForTerminal(term.height, reservedLines=reservedLines) + def buildServices(): # TODO: Move this into a dependency so that it can be executed with just a list of services. global dockerComposeServicesYaml try: - runPrebuildHook() + requiredIssues = requiredEnvironmentIssues(dockerComposeServicesYaml) + if requiredIssues: + print("") + print("Build warning: required environment variables are missing:") + for description in requiredIssues.values(): + print("* %s" % description) + print("Add the missing values to .env before building this stack.") + input("Press Enter to continue...") + return False + if not runPrebuildHook(): + return False menuStateFileYaml = {} menuStateFileYaml["services"] = dockerComposeServicesYaml @@ -75,13 +94,16 @@ def buildServices(): # TODO: Move this into a dependency so that it can be execu with open(r'%s' % dockerSavePathOutput, 'w') as outputFile: yaml.dump(menuStateFileYaml, outputFile) - runPostBuildHook() + if not runPostBuildHook(): + return False if os.path.exists('./postbuild.sh'): - servicesList = "" - for (index, serviceName) in enumerate(dockerComposeServicesYaml): - servicesList += " " + serviceName - subprocess.call("./postbuild.sh" + servicesList, shell=True) + postBuildCommand = ["./postbuild.sh"] + list(dockerComposeServicesYaml) + postBuildResult = subprocess.call(postBuildCommand) + if postBuildResult != 0: + print("postbuild.sh failed with exit status %s." % postBuildResult) + input("Press Enter to continue...") + return False return True except Exception as err: @@ -118,7 +140,7 @@ def generateLineText(text, textLength=None, paddingBefore=0, lineLength=26): return result - def renderHotZone(term, renderType, menu, selection, paddingBefore, allIssues): + def renderHotZone(term, renderType, menu, selection, paddingBefore): global paginationSize optionsLength = len(" >> Options ") optionsIssuesSpace = len(" ") @@ -126,7 +148,7 @@ def renderHotZone(term, renderType, menu, selection, paddingBefore, allIssues): spaceAfterissues = len(" ") issuesLength = len(" !! Issue ") - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") if paginationStartIndex >= 1: print(term.center("{b} {uaf} {uaf}{uaf}{uaf} {ual} {b}".format( @@ -137,18 +159,13 @@ def renderHotZone(term, renderType, menu, selection, paddingBefore, allIssues): else: print(term.center(commonEmptyLine(renderMode))) - menuItemsActiveRow = term.get_location()[0] if renderType == 2 or renderType == 1: # Rerender entire hotzone for (index, menuItem) in enumerate(menu): # Menu loop - if "issues" in menuItem[1] and menuItem[1]["issues"]: - allIssues.append({ "serviceName": menuItem[0], "issues": menuItem[1]["issues"] }) - if index >= paginationStartIndex and index < paginationStartIndex + paginationSize: lineText = generateLineText(menuItem[0], paddingBefore=paddingBefore) # Menu highlight logic if index == selection: - activeMenuLocation = term.get_location()[0] formattedLineText = '-> {t.blue_on_green}{title}{t.normal} <-'.format(t=term, title=menuItem[0]) paddedLineText = generateLineText(formattedLineText, textLength=len(menuItem[0]) + selectedTextLength, paddingBefore=paddingBefore - selectedTextLength) toPrint = paddedLineText @@ -211,8 +228,6 @@ def renderHotZone(term, renderType, menu, selection, paddingBefore, allIssues): print(renderOffsetCurrentSelection, lastSelection, renderOffsetLastSelection) lastSelection = selection - # menuItemsActiveRow - # activeMenuLocation if paginationStartIndex + paginationSize < len(menu): @@ -224,19 +239,48 @@ def renderHotZone(term, renderType, menu, selection, paddingBefore, allIssues): else: print(term.center(commonEmptyLine(renderMode))) + def getIssueEntries(): + return [ + (menuItem[0], issue, menuItem[1]["issues"][issue]) + for menuItem in menu + if menuItem[1].get("issues") + for issue in menuItem[1]["issues"] + ] + def mainRender(menu, selection, renderType = 1): global paginationStartIndex global paginationSize - paddingBefore = 4 - allIssues = [] + if not terminalSupportsMenu(term.width, term.height): + print(term.clear(), end="") + print(term.black_on_cornsilk4(term.center("IOTstack Build Menu"))) + print("") + print(term.center("Terminal is too small to render the build menu.")) + print(term.center("Resize to at least 82 columns by 30 rows, or press Escape.")) + return + + paddingBefore = 4 - if selection >= paginationStartIndex + paginationSize: - paginationStartIndex = selection - (paginationSize - 1) + 1 + issueEntries = getIssueEntries() + maximumIssueRows = min(4, max(1, term.height - 28)) + issueRows = compactIssueRows( + issueEntries, + contentWidth=76, + maximumRows=maximumIssueRows, + ) + + issuesHeight = issuePanelHeight(len(issueRows), fixedRows=8) + showHelpText = not hideHelpText and term.height >= 27 + issuesHeight + 1 + reservedLines = (27 if showHelpText else 19) + issuesHeight + newPaginationSize = pageSizeForTerminal(term.height, reservedLines=reservedLines) + if newPaginationSize != paginationSize: + paginationSize = newPaginationSize + paginationStartIndex = max(0, min(paginationStartIndex, max(0, len(menu) - paginationSize))) renderType = 1 - - if selection <= paginationStartIndex - 1: - paginationStartIndex = selection + + newPaginationStartIndex = paginationStart(selection, paginationStartIndex, paginationSize) + if newPaginationStartIndex != paginationStartIndex: + paginationStartIndex = newPaginationStartIndex renderType = 1 try: @@ -254,61 +298,64 @@ def mainRender(menu, selection, renderType = 1): print(term.center(commonEmptyLine(renderMode))) print(term.center(commonEmptyLine(renderMode))) - renderHotZone(term, renderType, menu, selection, paddingBefore, allIssues) + renderHotZone(term, renderType, menu, selection, paddingBefore) if (renderType == 1): print(term.center(commonEmptyLine(renderMode))) - if not hideHelpText: - room = term.height - (28 + len(allIssues) + paginationSize) - if room < 0: - allIssues.append({ "serviceName": "BuildStack Menu", "issues": { "screenSize": 'Not enough scren height to render correctly (t-height = ' + str(term.height) + ' v-lines = ' + str(room) + ')' } }) - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Not enough vertical room to render controls help text ({th}, {rm}) {bv}".format(bv=specialChars[renderMode]["borderVertical"], th=padText(str(term.height), 3), rm=padText(str(room), 3)))) - print(term.center(commonEmptyLine(renderMode))) - else: - print(term.center(commonEmptyLine(renderMode))) - print(term.center("{bv} Controls: {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Space] to select or deselect image {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Up] and [Down] to move selection cursor {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Right] for options for containers that support them {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Tab] Expand or collapse build menu size {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [H] Show/hide this text {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - # print(term.center("{bv} [F] Filter options {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Enter] to begin build {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center("{bv} [Escape] to cancel build {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) - print(term.center(commonEmptyLine(renderMode))) - print(term.center(commonEmptyLine(renderMode))) + if showHelpText: + print(term.center(commonEmptyLine(renderMode))) + print(term.center("{bv} Controls: {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) + print(term.center("{bv} [Space] to select or deselect image {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) + print(term.center("{bv} [Up] and [Down] to move selection cursor {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) + print(term.center("{bv} [Right] for options for containers that support them {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) + print(term.center("{bv} [H] Show/hide this text {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) + print(term.center("{bv} [Enter] to begin build {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) + print(term.center("{bv} [Escape] to cancel build {bv}".format(bv=specialChars[renderMode]["borderVertical"]))) + print(term.center(commonEmptyLine(renderMode))) + if transientMessage: + print(term.center(commonTextLine( + renderMode, + transientMessage, + paddingBefore=6, + style=term.yellow, + ))) + else: + print(term.center(commonEmptyLine(renderMode))) print(term.center(commonEmptyLine(renderMode))) print(term.center(commonBottomBorder(renderMode))) - if len(allIssues) > 0: + if issueEntries: print(term.center("")) print(term.center("")) print(term.center("")) - print(term.center(("{btl}{bh}{bh}{bh}{bh}{bh}{bh} Build Issues " - "{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}" - "{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}" - "{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}" - "{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}" - "{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}" - "{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}" - "{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}{bh}" - "{bh}{bh}{bh}{bh}{bh}{bh}{bh}{btr}").format( - btl=specialChars[renderMode]["borderTopLeft"], - btr=specialChars[renderMode]["borderTopRight"], - bh=specialChars[renderMode]["borderHorizontal"] + issueBoxWidth = 80 + issueTitle = " Build Issues " + leftBorderSize = 6 + rightBorderSize = issueBoxWidth - leftBorderSize - len(issueTitle) + print(term.center( + specialChars[renderMode]["borderTopLeft"] + + (specialChars[renderMode]["borderHorizontal"] * leftBorderSize) + + issueTitle + + (specialChars[renderMode]["borderHorizontal"] * rightBorderSize) + + specialChars[renderMode]["borderTopRight"] + )) + print(term.center(commonEmptyLine(renderMode, size=issueBoxWidth))) + for issueRow in issueRows: + print(term.center(commonTextLine( + renderMode, + issueRow, + size=issueBoxWidth, + paddingBefore=2, + ))) + print(term.center(commonTextLine( + renderMode, + "[I] View all issues", + size=issueBoxWidth, + paddingBefore=2, + style=term.yellow, ))) - print(term.center(commonEmptyLine(renderMode, size = 139))) - for serviceIssues in allIssues: - for index, issue in enumerate(serviceIssues["issues"]): - spacesAndBracketsLen = 5 - issueAndTypeLen = len(issue) + len(serviceIssues["serviceName"]) + spacesAndBracketsLen - serviceNameAndConflictType = '{t.red_on_black}{issueService}{t.normal} ({t.yellow_on_black}{issueType}{t.normal}) '.format(t=term, issueService=serviceIssues["serviceName"], issueType=issue) - formattedServiceNameAndConflictType = generateLineText(str(serviceNameAndConflictType), textLength=issueAndTypeLen, paddingBefore=0, lineLength=32) - issueDescription = generateLineText(str(serviceIssues["issues"][issue]), textLength=len(str(serviceIssues["issues"][issue])), paddingBefore=0, lineLength=103) - print(term.center("{bv} {nm} - {desc} {bv}".format(nm=formattedServiceNameAndConflictType, desc=issueDescription, bv=specialChars[renderMode]["borderVertical"]) )) - print(term.center(commonEmptyLine(renderMode, size = 139))) - print(term.center(commonBottomBorder(renderMode, size = 139))) + print(term.center(commonEmptyLine(renderMode, size=issueBoxWidth))) + print(term.center(commonBottomBorder(renderMode, size=issueBoxWidth))) except Exception as err: print("There was an error rendering the menu:") @@ -329,34 +376,16 @@ def loadAllServices(reload = False): global dockerComposeServicesYaml dockerComposeServicesYaml.clear() for (index, checkedMenuItem) in enumerate(checkedMenuItems): - if reload == False: - if not checkedMenuItem in dockerComposeServicesYaml: - serviceFilePath = templatesDirectory + '/' + checkedMenuItem + '/' + servicesFileName - with open(r'%s' % serviceFilePath) as yamlServiceFile: - dockerComposeServicesYaml[checkedMenuItem] = yaml.load(yamlServiceFile)[checkedMenuItem] - else: - print("reload!") - time.sleep(1) - serviceFilePath = templatesDirectory + '/' + checkedMenuItem + '/' + servicesFileName - with open(r'%s' % serviceFilePath) as yamlServiceFile: - dockerComposeServicesYaml[checkedMenuItem] = yaml.load(yamlServiceFile)[checkedMenuItem] + templateServices = loadServiceTemplate(yaml, templatesDirectory, checkedMenuItem, servicesFileName) + mergeServiceTemplate(dockerComposeServicesYaml, templateServices, reload=reload) return True def loadService(serviceName, reload = False): try: global dockerComposeServicesYaml - if reload == False: - if not serviceName in dockerComposeServicesYaml: - serviceFilePath = templatesDirectory + '/' + serviceName + '/' + servicesFileName - with open(r'%s' % serviceFilePath) as yamlServiceFile: - dockerComposeServicesYaml[serviceName] = yaml.load(yamlServiceFile)[serviceName] - else: - print("reload!") - time.sleep(1) - servicesFileNamePath = templatesDirectory + '/' + serviceName + '/' + servicesFileName - with open(r'%s' % serviceFilePath) as yamlServiceFile: - dockerComposeServicesYaml[serviceName] = yaml.load(yamlServiceFile)[serviceName] + templateServices = loadServiceTemplate(yaml, templatesDirectory, serviceName, servicesFileName) + mergeServiceTemplate(dockerComposeServicesYaml, templateServices, reload=reload) except Exception as err: print("Error running build menu:", err) print("Check the following:") @@ -369,151 +398,176 @@ def loadService(serviceName, reload = False): return True + def createHookContext(serviceName): + return HookContext( + services=dockerComposeServicesYaml, + serviceName=serviceName, + renderMode=renderMode, + terminal=term, + ) + + def getOwnedServices(serviceName): + templateServices = loadServiceTemplate( + yaml, templatesDirectory, serviceName, servicesFileName + ) + return { + ownedServiceName: dockerComposeServicesYaml[ownedServiceName] + for ownedServiceName in templateServices + if ownedServiceName in dockerComposeServicesYaml + } + def checkForIssues(): global dockerComposeServicesYaml for (index, checkedMenuItem) in enumerate(checkedMenuItems): + menuItemIndex = getMenuItemIndexByService(checkedMenuItem) buildScriptPath = templatesDirectory + '/' + checkedMenuItem + '/' + buildScriptFile - if os.path.exists(buildScriptPath): - try: - with open(buildScriptPath, "rb") as pythonDynamicImportFile: - code = compile(pythonDynamicImportFile.read(), buildScriptPath, "exec") - execGlobals = { - "dockerComposeServicesYaml": dockerComposeServicesYaml, - "toRun": "checkForRunChecksHook", - "currentServiceName": checkedMenuItem - } - execLocals = locals() - exec(code, execGlobals, execLocals) - if "buildHooks" in execGlobals and "runChecksHook" in execGlobals["buildHooks"] and execGlobals["buildHooks"]["runChecksHook"]: - execGlobals = { - "dockerComposeServicesYaml": dockerComposeServicesYaml, - "toRun": "runChecks", - "currentServiceName": checkedMenuItem - } - execLocals = locals() - try: - exec(code, execGlobals, execLocals) - if "issues" in execGlobals and len(execGlobals["issues"]) > 0: - menu[getMenuItemIndexByService(checkedMenuItem)][1]["issues"] = execGlobals["issues"] - else: - menu[getMenuItemIndexByService(checkedMenuItem)][1]["issues"] = [] - except Exception as err: - print("Error running checkForIssues on '%s'" % checkedMenuItem) - traceback.print_exc() - input("Press Enter to continue...") - else: - menu[getMenuItemIndexByService(checkedMenuItem)][1]["issues"] = [] - except Exception as err: - print("Error running checkForIssues on '%s'" % checkedMenuItem) - traceback.print_exc() - input("Press any key to exit...") - sys.exit(1) + + try: + issues = {} + if os.path.exists(buildScriptPath): + context = createHookContext(checkedMenuItem) + if serviceHookAvailable(buildScriptPath, "runChecks", context): + hookIssues = runServiceHook(buildScriptPath, "runChecks", context) + issues.update(hookIssues) + dockerComposeServicesYaml = context.services + + ownedServices = getOwnedServices(checkedMenuItem) + issues.update(requiredEnvironmentIssues(ownedServices)) + menu[menuItemIndex][1]["issues"] = issues if issues else [] + except Exception: + print("Error running checkForIssues on '%s'" % checkedMenuItem) + traceback.print_exc() + input("Press any key to exit...") + sys.exit(1) def checkForOptions(): - global dockerComposeServicesYaml for (index, menuItem) in enumerate(menu): - buildScriptPath = templatesDirectory + '/' + menuItem[0] + '/' + buildScriptFile - if os.path.exists(buildScriptPath): - try: - with open(buildScriptPath, "rb") as pythonDynamicImportFile: - code = compile(pythonDynamicImportFile.read(), buildScriptPath, "exec") - execGlobals = { - "dockerComposeServicesYaml": dockerComposeServicesYaml, - "toRun": "checkForOptionsHook", - "currentServiceName": menuItem[0], - "renderMode": renderMode - } - execLocals = {} - exec(code, execGlobals, execLocals) - if not "buildHooks" in menu[getMenuItemIndexByService(menuItem[0])][1]: - menu[getMenuItemIndexByService(menuItem[0])][1]["buildHooks"] = {} - if "options" in execGlobals["buildHooks"] and execGlobals["buildHooks"]["options"]: - menu[getMenuItemIndexByService(menuItem[0])][1]["buildHooks"]["options"] = True - except Exception as err: - print("Error running checkForOptions on '%s'" % menuItem[0]) - traceback.print_exc() - input("Press any key to exit...") - sys.exit(1) + serviceName = menuItem[0] + buildScriptPath = templatesDirectory + '/' + serviceName + '/' + buildScriptFile + hookState = menuItem[1].setdefault("buildHooks", {}) + hookState["serviceOptions"] = False + + try: + templateServices = ( + loadServiceTemplate( + yaml, templatesDirectory, serviceName, servicesFileName + ) + if menuItem[1]["checked"] + else {} + ) + hookState["environmentOptions"] = bool( + configurableEnvironmentVariables(templateServices) + ) + if os.path.exists(buildScriptPath): + context = createHookContext(serviceName) + hookState["serviceOptions"] = serviceHookAvailable( + buildScriptPath, "options", context + ) + hookState["options"] = ( + hookState["environmentOptions"] or hookState["serviceOptions"] + ) + except Exception: + print("Error checking service options on '%s'" % serviceName) + traceback.print_exc() + input("Press any key to exit...") + sys.exit(1) def runPrebuildHook(): global dockerComposeServicesYaml for (index, checkedMenuItem) in enumerate(checkedMenuItems): buildScriptPath = templatesDirectory + '/' + checkedMenuItem + '/' + buildScriptFile - if os.path.exists(buildScriptPath): - with open(buildScriptPath, "rb") as pythonDynamicImportFile: - code = compile(pythonDynamicImportFile.read(), buildScriptPath, "exec") - execGlobals = { - "dockerComposeServicesYaml": dockerComposeServicesYaml, - "toRun": "checkForPreBuildHook", - "currentServiceName": checkedMenuItem - } - execLocals = locals() - try: - exec(code, execGlobals, execLocals) - if "preBuildHook" in execGlobals["buildHooks"] and execGlobals["buildHooks"]["preBuildHook"]: - execGlobals = { - "dockerComposeServicesYaml": dockerComposeServicesYaml, - "toRun": "preBuild", - "currentServiceName": checkedMenuItem - } - execLocals = locals() - exec(code, execGlobals, execLocals) - except Exception as err: - print("Error running PreBuildHook on '%s'" % checkedMenuItem) - traceback.print_exc() + if not os.path.exists(buildScriptPath): + continue + + try: + context = createHookContext(checkedMenuItem) + if serviceHookAvailable(buildScriptPath, "preBuild", context): + hookResult = runServiceHook(buildScriptPath, "preBuild", context) + if hookResult is False: + print("preBuild reported a failure for '%s'." % checkedMenuItem) input("Press Enter to continue...") - try: # If the prebuild hook modified the docker-compose object, pull it from the script back to here. - dockerComposeServicesYaml = execGlobals["dockerComposeServicesYaml"] - except: - pass + return False + dockerComposeServicesYaml = context.services + except Exception: + print("Error running preBuild on '%s'" % checkedMenuItem) + traceback.print_exc() + input("Press Enter to continue...") + return False + return True def runPostBuildHook(): + global dockerComposeServicesYaml for (index, checkedMenuItem) in enumerate(checkedMenuItems): buildScriptPath = templatesDirectory + '/' + checkedMenuItem + '/' + buildScriptFile - if os.path.exists(buildScriptPath): - with open(buildScriptPath, "rb") as pythonDynamicImportFile: - code = compile(pythonDynamicImportFile.read(), buildScriptPath, "exec") - execGlobals = { - "dockerComposeServicesYaml": dockerComposeServicesYaml, - "toRun": "checkForPostBuildHook", - "currentServiceName": checkedMenuItem - } - execLocals = locals() - try: - exec(code, execGlobals, execLocals) - if "postBuildHook" in execGlobals["buildHooks"] and execGlobals["buildHooks"]["postBuildHook"]: - execGlobals = { - "dockerComposeServicesYaml": dockerComposeServicesYaml, - "toRun": "postBuild", - "currentServiceName": checkedMenuItem - } - execLocals = locals() - exec(code, execGlobals, execLocals) - except Exception as err: - print("Error running PostBuildHook on '%s'" % checkedMenuItem) - traceback.print_exc() + if not os.path.exists(buildScriptPath): + continue + + try: + context = createHookContext(checkedMenuItem) + if serviceHookAvailable(buildScriptPath, "postBuild", context): + hookResult = runServiceHook(buildScriptPath, "postBuild", context) + if hookResult is False: + print("postBuild reported a failure for '%s'." % checkedMenuItem) input("Press Enter to continue...") + return False + dockerComposeServicesYaml = context.services + except Exception: + print("Error running postBuild on '%s'" % checkedMenuItem) + traceback.print_exc() + input("Press Enter to continue...") + return False + return True def executeServiceOptions(): global dockerComposeServicesYaml menuItem = menu[selection] - if menu[selection][1]["checked"] and "buildHooks" in menuItem[1] and "options" in menuItem[1]["buildHooks"] and menuItem[1]["buildHooks"]["options"]: - buildScriptPath = templatesDirectory + '/' + menuItem[0] + '/' + buildScriptFile - if os.path.exists(buildScriptPath): - with open(buildScriptPath, "rb") as pythonDynamicImportFile: - code = compile(pythonDynamicImportFile.read(), buildScriptPath, "exec") - - execGlobals = { - "dockerComposeServicesYaml": dockerComposeServicesYaml, - "toRun": "runOptionsMenu", - "currentServiceName": menuItem[0], - "renderMode": renderMode - } - execLocals = locals() - exec(code, execGlobals, execLocals) - dockerComposeServicesYaml = execGlobals["dockerComposeServicesYaml"] - checkForIssues() - mainRender(menu, selection, 1) + hookState = menuItem[1].get("buildHooks", {}) + if not menuItem[1]["checked"] or not hookState.get("options", False): + return + + serviceName = menuItem[0] + buildScriptPath = templatesDirectory + '/' + serviceName + '/' + buildScriptFile + templateServices = loadServiceTemplate( + yaml, templatesDirectory, serviceName, servicesFileName + ) + + def onEnvironmentValueSaved(name, unusedValue): + restoreEnvironmentVariableReferences( + dockerComposeServicesYaml, + templateServices, + name, + ) + + def openCustomServiceOptions(): + global dockerComposeServicesYaml + context = createHookContext(serviceName) + result = runServiceHook(buildScriptPath, "options", context) + dockerComposeServicesYaml = context.services + return result + + try: + if hookState.get("environmentOptions", False): + customOptions = ( + openCustomServiceOptions + if hookState.get("serviceOptions", False) + else None + ) + runEnvironmentOptions( + term, + renderMode, + serviceName, + templateServices, + openServiceOptions=customOptions, + onValueSaved=onEnvironmentValueSaved, + ) + elif hookState.get("serviceOptions", False): + openCustomServiceOptions() + checkForIssues() + mainRender(menu, selection, 1) + except Exception: + print("Error running service options on '%s'" % serviceName) + traceback.print_exc() + input("Press Enter to continue...") def getMenuItemIndexByService(serviceName): for (index, menuItem) in enumerate(menu): @@ -525,7 +579,8 @@ def checkMenuItem(selection): if menu[selection][1]["checked"] == True: menu[selection][1]["checked"] = False menu[selection][1]["issues"] = None - del dockerComposeServicesYaml[menu[selection][0]] + templateServices = loadServiceTemplate(yaml, templatesDirectory, menu[selection][0], servicesFileName) + removeServiceTemplate(dockerComposeServicesYaml, templateServices) else: menu[selection][1]["checked"] = True print(menu[selection][0]) @@ -533,7 +588,11 @@ def checkMenuItem(selection): def prepareMenuState(): global dockerComposeServicesYaml - for (index, serviceName) in enumerate(dockerComposeServicesYaml): + selectedTemplates = [ + serviceName for serviceName in dockerComposeServicesYaml + if serviceName in templatesList + ] + for serviceName in selectedTemplates: checkMenuItem(getMenuItemIndexByService(serviceName)) setCheckedMenuItems() checkForIssues() @@ -548,17 +607,18 @@ def loadCurrentConfigs(templatesList): previousConfigs = yaml.load(fileSavedConfigs) if not previousConfigs == None: if "services" in previousConfigs: - dockerComposeServicesYaml = {} - for (index, serviceName) in enumerate(previousConfigs["services"]): - if serviceName in templatesList: # This ensures every service loaded has a template directory - dockerComposeServicesYaml[serviceName] = previousConfigs["services"][serviceName] - return True + dockerComposeServicesYaml = restoreSavedServiceTemplates( + yaml, + templatesDirectory, + templatesList, + previousConfigs["services"], + servicesFileName, + ) + return bool(dockerComposeServicesYaml) dockerComposeServicesYaml = {} return False def onResize(sig, action): - global paginationToggle - paginationToggle = [10, term.height - 25] mainRender(menu, selection, 1) templatesList = generateTemplateList(templatesDirectoryFolders) @@ -576,26 +636,33 @@ def onResize(sig, action): if loadCurrentConfigs(templatesList): prepareMenuState() mainRender(menu, selection, 1) + needsRender = 0 selectionInProgress = True with term.cbreak(): while selectionInProgress: key = term.inkey(esc_delay=0.05) + if key and transientMessage: + transientMessage = None + needsRender = 1 if key.is_sequence: - if key.name == 'KEY_TAB': - needsRender = 1 - if paginationSize == paginationToggle[0]: - paginationSize = paginationToggle[1] - paginationStartIndex = 0 - else: - paginationSize = paginationToggle[0] if key.name == 'KEY_DOWN': selection += 1 - needsRender = 2 + if needsRender != 1: + needsRender = 2 if key.name == 'KEY_UP': selection -= 1 - needsRender = 2 + if needsRender != 1: + needsRender = 2 if key.name == 'KEY_RIGHT': - executeServiceOptions() + hasOptions = menu[selection][1].get("buildHooks", {}).get("options", False) + transientMessage = serviceOptionsMessage( + hasOptions, + menu[selection][1]["checked"], + ) + if transientMessage: + needsRender = 1 + else: + executeServiceOptions() if key.name == 'KEY_ENTER': setCheckedMenuItems() checkForIssues() @@ -617,14 +684,18 @@ def onResize(sig, action): else: hideHelpText = True needsRender = 1 - else: - print(key) - time.sleep(0.5) + elif key.lower() == 'i': + issueEntries = getIssueEntries() + if issueEntries: + runIssueViewer(term, renderMode, issueEntries) + needsRender = 1 selection = selection % len(menu) - mainRender(menu, selection, needsRender) + if needsRender > 0: + mainRender(menu, selection, needsRender) + needsRender = 0 -originalSignalHandler = signal.getsignal(signal.SIGINT) +originalSignalHandler = signal.getsignal(signal.SIGWINCH) main() signal.signal(signal.SIGWINCH, originalSignalHandler) diff --git a/scripts/default_ports_md_generator.py b/scripts/default_ports_md_generator.py new file mode 100755 index 000000000..bccb0e144 --- /dev/null +++ b/scripts/default_ports_md_generator.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import pathlib +import re + + +def readServiceDetails(serviceFile): + source = serviceFile.read_text() + nameMatch = re.search( + r"^\s*container_name:\s*[\"\x27]?([A-Za-z0-9_.-]+)", source, re.MULTILINE + ) + serviceName = nameMatch.group(1) if nameMatch else "Parsing error" + mode = "host" if re.search( + r"^\s*network_mode:\s*[\"\x27]?host[\"\x27]?\s*$", + source, + re.MULTILINE, + ) else "non-host" + + ports = [] + listValues = re.findall( + r"^\s*-\s*[\"\x27]?([^\"\x27#\s]+)[\"\x27]?\s*(?:#.*)?$", + source, + re.MULTILINE, + ) + for value in listValues: + portValue = value.split("/", 1)[0] + parts = portValue.rsplit(":", 2) + if len(parts) >= 2 and parts[-2].isdigit() and parts[-1].isdigit(): + port = "%s:%s" % (parts[-2], parts[-1]) + if port not in ports: + ports.append(port) + + return serviceName, mode, ports + + +def main(): + print("| Service Name | Mode | Port(s)
*External:Internal* |") + print("| ------------ | -----| --------------- |") + + repositoryRoot = pathlib.Path(__file__).resolve().parents[1] + for serviceFile in sorted(repositoryRoot.glob(".templates/*/service.yml")): + serviceName, mode, ports = readServiceDetails(serviceFile) + portText = "".join("%s
" % port for port in ports) + print("| %s | %s | %s|" % (serviceName, mode, portText)) + + +if __name__ == "__main__": + main() diff --git a/scripts/default_ports_md_generator.sh b/scripts/default_ports_md_generator.sh deleted file mode 100755 index 92476a791..000000000 --- a/scripts/default_ports_md_generator.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 - -# This script will return a markdown table containing the service names, mode (host or non-host), and default -# external ports used by all services found in the .templates directory. The markdown output can be used to -# quickly update the docs/Basic_setup/Default-Configs.md file. - -import glob -import pathlib -import re - -# Setup columns & print service names, mode, and default ports. - -print("| Service Name | Mode | Port(s)
*External:Internal* |") -print("| ------------ | -----| --------------- |") - -# Change directories - -currentPath = pathlib.Path(__file__) -dirName = str(currentPath.parents[1]) -templates = glob.glob(dirName + '/.templates/**/service.yml',recursive = True) - -# Iterate through service.ymls for required info. - -for template in sorted(templates): - - with open(template) as file: - - fileInput = file.read() - - # Search for service names and mode. - - try: - serviceName = re.search(r'container_name:.?(["a-z0-9_-]+)', fileInput).group(1) - except: - serviceName = 'Parsing error' - try: - if (re.search(r'^([^\#]\s+network_mode:).?([a-z0-9]+)', fileInput,flags = re.M).group(2) == 'host'): - mode = 'host' - except: - mode = 'non-host' - - # Print service and mode but do not end the line. - - print("| " + serviceName + " | " + mode + " | ", end= "") - - # Search for ports used by each service. findall is split into 2 groups to deal with #'s in some service.yml's. - # Keep only the ports and not the whitespace or "-" - - portSearchResult = re.findall(r'^(\s*[-]\s*"*)(\d{2,5}[:]\d{2,5})', fileInput,re.M) - ports = [] - for result in portSearchResult: - ports.append(result[1]) - - # Get rid of found doubles - UDP and TCP ports etc. - - dropDuplicates = [] - [dropDuplicates.append(port) for port in ports if port not in dropDuplicates] - - # Print the ports used and end the line when the for loop completes. - - for port in dropDuplicates: - print(port + "
", end= "") - print("|") \ No newline at end of file diff --git a/scripts/deps/chars.py b/scripts/deps/chars.py index 51cfa0d77..db3bae8e2 100755 --- a/scripts/deps/chars.py +++ b/scripts/deps/chars.py @@ -70,3 +70,22 @@ def commonEmptyLine(renderMode, size=80): output += " " output += "{bv}".format(bv=specialChars[renderMode]["borderVertical"]) return output + +def commonTextLine(renderMode, text, size=80, paddingBefore=0, style=None): + """Render text inside a fixed-width menu border. + + Styling is applied after padding is calculated so terminal escape sequences do + not change the visible width of the row. + """ + text = str(text) + availableWidth = max(0, size - paddingBefore) + text = text[:availableWidth] + styledText = style(text) if style else text + paddingAfter = size - paddingBefore - len(text) + border = specialChars[renderMode]["borderVertical"] + return "{border}{before}{text}{after}{border}".format( + border=border, + before=" " * paddingBefore, + text=styledText, + after=" " * paddingAfter, + ) diff --git a/scripts/deps/compose_environment.py b/scripts/deps/compose_environment.py new file mode 100644 index 000000000..0ff3662e6 --- /dev/null +++ b/scripts/deps/compose_environment.py @@ -0,0 +1,278 @@ +import pathlib +import os +import re +import stat + + +_ENVIRONMENT_VARIABLE = re.compile( + r"(?= 2 and value[0] == value[-1] and value[0] in ("'", '"'): + quote = value[0] + value = value[1:-1] + if quote == '"': + value = _decodeDoubleQuotedDotEnvValue(value) + elif " #" in value: + value = value.split(" #", 1)[0].rstrip() + environment[name] = value + + return environment + + +def configurableEnvironmentVariables(value): + variables = {} + + def inspect(item): + if isinstance(item, dict): + for child in item.values(): + inspect(child) + return + if isinstance(item, (list, tuple)): + for child in item: + inspect(child) + return + if not isinstance(item, str): + return + + for match in _ENVIRONMENT_VARIABLE.finditer(item): + name, operator, message = match.groups() + operator = operator or "" + message = (message or "").strip() + shouldConfigure = operator in (":?", "?") or isSensitiveEnvironmentName(name) + if not shouldConfigure: + continue + if name not in variables: + variables[name] = { + "operator": operator, + "message": message, + } + else: + operatorPriority = { + "": 0, "-": 0, ":-": 0, "+": 0, ":+": 0, + "?": 1, ":?": 2, + } + existingOperator = variables[name]["operator"] + if operatorPriority[operator] > operatorPriority[existingOperator]: + variables[name]["operator"] = operator + variables[name]["message"] = message + + inspect(value) + return variables + + +def restoreEnvironmentVariableReferences(currentValue, templateValue, name): + """Restore template interpolation expressions for one environment variable. + + Saved stack files may contain values that Compose already interpolated. Those + literal values are useful for restoring the rest of a service configuration, + but they hide the setting from the options menu and prevent a newly saved + .env value from taking effect. This mutates only template paths which refer + to ``name`` and leaves unrelated restored settings alone. + """ + changed = 0 + + def usesVariable(value): + if not isinstance(value, str): + return False + return any( + match.group(1) == name + for match in _ENVIRONMENT_VARIABLE.finditer(value) + ) + + def assignmentName(value): + if not isinstance(value, str) or "=" not in value: + return None + candidate = value.split("=", 1)[0] + if re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", candidate): + return candidate + return None + + def restore(current, template): + nonlocal changed + + if isinstance(template, dict): + if not isinstance(current, dict): + return + for key, templateChild in template.items(): + if key not in current: + if configurableEnvironmentVariables(templateChild).get(name): + current[key] = templateChild + changed += 1 + continue + if usesVariable(templateChild): + if current[key] != templateChild: + current[key] = templateChild + changed += 1 + else: + restore(current[key], templateChild) + return + + if isinstance(template, (list, tuple)): + if not isinstance(current, list): + return + for templateIndex, templateChild in enumerate(template): + if usesVariable(templateChild): + targetIndex = None + variableAssignment = assignmentName(templateChild) + if variableAssignment is not None: + for currentIndex, currentChild in enumerate(current): + if assignmentName(currentChild) == variableAssignment: + targetIndex = currentIndex + break + if targetIndex is None and templateIndex < len(current): + targetIndex = templateIndex + if targetIndex is None: + current.append(templateChild) + changed += 1 + elif current[targetIndex] != templateChild: + current[targetIndex] = templateChild + changed += 1 + elif templateIndex < len(current): + restore(current[templateIndex], templateChild) + + restore(currentValue, templateValue) + return changed + + +def requiredEnvironmentVariables(value): + return { + name: requirement + for name, requirement in configurableEnvironmentVariables(value).items() + if requirement["operator"] in (":?", "?") + } + + +def requiredEnvironmentIssues(value, envPath=".env", processEnvironment=None): + environment = loadDotEnv(envPath) + if processEnvironment is None: + processEnvironment = os.environ + environment.update(processEnvironment) + + issues = {} + for name, requirement in requiredEnvironmentVariables(value).items(): + isMissing = name not in environment + if requirement["operator"] == ":?": + isMissing = isMissing or environment.get(name, "") == "" + if isMissing: + issues["missingEnvironment:%s" % name] = ( + "%s is required. Open Options to configure it." % name + ) + return issues + + +def isSensitiveEnvironmentName(name): + upperName = name.upper() + return any(marker in upperName for marker in ( + "PASSWORD", + "PASSWD", + "SECRET", + "TOKEN", + "AUTHORIZATION", + "API_KEY", + )) + +def isPasswordEnvironmentName(name): + upperName = name.upper() + return "PASSWORD" in upperName or "PASSWD" in upperName + + + +def suggestedEnvironmentValue(name, message): + match = re.search(r"(?:^|\s)%s=([^\s]+)" % re.escape(name), message) + return match.group(1) if match else "" + +def defaultEnvironmentValue(name, requirement): + if requirement["operator"] in (":-", "-"): + return requirement["message"] + suggested = suggestedEnvironmentValue(name, requirement["message"]) + if suggested.startswith("%random"): + return "" + return suggested + + +def _encodeDotEnvValue(value): + if "\n" in value or "\r" in value: + raise ValueError("Environment values must fit on one line") + if re.match(r"^[A-Za-z0-9_./:@%+,-]*$", value): + return value + escaped = value.replace("\\", "\\\\").replace('"', '\\"').replace("$", "$$") + return '"%s"' % escaped + + +def setDotEnvValue(path, name, value): + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", name): + raise ValueError("Invalid environment variable name") + + path = pathlib.Path(path) + existingMode = path.stat().st_mode if path.exists() else None + lines = path.read_text().splitlines(True) if path.exists() else [] + assignmentPattern = re.compile( + r"^\s*(?:export\s+)?%s\s*=" % re.escape(name) + ) + replacement = "%s=%s" % (name, _encodeDotEnvValue(value)) + replaced = False + + for index, line in enumerate(lines): + if assignmentPattern.match(line): + ending = "\n" if line.endswith("\n") else "" + lines[index] = replacement + ending + replaced = True + break + + if not replaced: + if lines and not lines[-1].endswith("\n"): + lines[-1] += "\n" + lines.append(replacement + "\n") + + path.parent.mkdir(parents=True, exist_ok=True) + temporaryPath = path.with_name(path.name + ".tmp") + temporaryPath.write_text("".join(lines)) + if existingMode is not None: + os.chmod(str(temporaryPath), stat.S_IMODE(existingMode)) + else: + os.chmod(str(temporaryPath), 0o600) + os.replace(str(temporaryPath), str(path)) diff --git a/scripts/deps/environment_options.py b/scripts/deps/environment_options.py new file mode 100644 index 000000000..90eb4b62c --- /dev/null +++ b/scripts/deps/environment_options.py @@ -0,0 +1,569 @@ +import os +import secrets +import signal +import string +import textwrap + +from deps.chars import ( + commonBottomBorder, + commonEmptyLine, + commonTextLine, + commonTopBorder, +) +from deps.compose_environment import ( + configurableEnvironmentVariables, + defaultEnvironmentValue, + isPasswordEnvironmentName, + isSensitiveEnvironmentName, + loadDotEnv, + requiredEnvironmentIssues, + setDotEnvValue, +) + + +def generateEnvironmentSecret(size=32): + alphabet = string.ascii_letters + string.digits + return "".join(secrets.choice(alphabet) for unusedIndex in range(size)) + + +def _boxWidth(term): + return min(80, max(40, term.width - 2)) + + +def _runMenu(term, renderMode, title, prompt, entries): + state = { + "selection": 0, + "offset": 0, + } + + def dimensions(): + visibleRows = max(1, min(len(entries), term.height - 9)) + if state["selection"] < state["offset"]: + state["offset"] = state["selection"] + if state["selection"] >= state["offset"] + visibleRows: + state["offset"] = state["selection"] - visibleRows + 1 + maximumOffset = max(0, len(entries) - visibleRows) + state["offset"] = max(0, min(state["offset"], maximumOffset)) + return _boxWidth(term), visibleRows + + def render(): + boxWidth, visibleRows = dimensions() + visibleEntries = entries[state["offset"]:state["offset"] + visibleRows] + print(term.clear(), end="") + print(term.black_on_cornsilk4(term.center(title))) + print("") + print(term.center(commonTopBorder(renderMode, size=boxWidth))) + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + print(term.center(commonTextLine( + renderMode, + prompt, + size=boxWidth, + paddingBefore=3, + ))) + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + for visibleIndex, unusedEntry in enumerate(visibleEntries): + entryIndex = state["offset"] + visibleIndex + label = entries[entryIndex][1] + selected = entryIndex == state["selection"] + displayLabel = "-> %s <-" % label if selected else " %s" % label + print(term.center(commonTextLine( + renderMode, + displayLabel, + size=boxWidth, + paddingBefore=3, + style=term.blue_on_green if selected else None, + ))) + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + if len(entries) > visibleRows: + first = state["offset"] + 1 + last = state["offset"] + len(visibleEntries) + status = "Items %s-%s of %s" % (first, last, len(entries)) + else: + status = "" + print(term.center(commonTextLine( + renderMode, + status, + size=boxWidth, + paddingBefore=3, + ))) + print(term.center(commonTextLine( + renderMode, + "[Up/Down] Move [Enter] Select [Esc] Back", + size=boxWidth, + paddingBefore=3, + ))) + print(term.center(commonBottomBorder(renderMode, size=boxWidth))) + + def onResize(sig, action): + render() + + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + render() + while True: + key = term.inkey(esc_delay=0.05) + if key.name == "KEY_ESCAPE" or ( + not key.is_sequence and str(key).lower() == "q" + ): + return None + if key.name == "KEY_UP": + state["selection"] = (state["selection"] - 1) % len(entries) + render() + elif key.name == "KEY_DOWN": + state["selection"] = (state["selection"] + 1) % len(entries) + render() + elif key.name == "KEY_HOME": + state["selection"] = 0 + render() + elif key.name == "KEY_END": + state["selection"] = len(entries) - 1 + render() + elif key.name == "KEY_PGUP": + unusedWidth, visibleRows = dimensions() + state["selection"] = max(0, state["selection"] - visibleRows) + render() + elif key.name == "KEY_PGDOWN": + unusedWidth, visibleRows = dimensions() + state["selection"] = min( + len(entries) - 1, + state["selection"] + visibleRows, + ) + render() + elif key.name == "KEY_ENTER": + return entries[state["selection"]][0] + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) + + +def _storeValue(envPath, name, value, onValueSaved): + setDotEnvValue(envPath, name, value) + if onValueSaved is not None: + onValueSaved(name, value) + + +def _promptForValue( + term, + renderMode, + serviceName, + name, + requirement, + envPath, + onValueSaved, +): + sensitive = isSensitiveEnvironmentName(name) + currentValue = loadDotEnv(envPath).get(name, "") + suggestedValue = defaultEnvironmentValue(name, requirement) + initialValue = "" if sensitive else currentValue or suggestedValue + state = { + "value": initialValue, + "message": None, + } + + def render(): + boxWidth = _boxWidth(term) + displayValue = "*" * len(state["value"]) if sensitive else state["value"] + maximumValueWidth = max(1, boxWidth - 12) + displayValue = displayValue[-maximumValueWidth:] + print(term.clear(), end="") + print(term.black_on_cornsilk4(term.center("%s Setting" % serviceName))) + print("") + print(term.center(commonTopBorder(renderMode, size=boxWidth))) + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + print(term.center(commonTextLine( + renderMode, + "Enter a value for %s:" % name, + size=boxWidth, + paddingBefore=3, + ))) + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + print(term.center(commonTextLine( + renderMode, + "Value: %s" % displayValue, + size=boxWidth, + paddingBefore=3, + style=term.yellow, + ))) + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + print(term.center(commonTextLine( + renderMode, + state["message"] or "", + size=boxWidth, + paddingBefore=3, + style=term.red if state["message"] else None, + ))) + print(term.center(commonTextLine( + renderMode, + "[Enter] Save [Backspace] Delete [Ctrl+U] Clear [Esc] Cancel", + size=boxWidth, + paddingBefore=3, + ))) + print(term.center(commonBottomBorder(renderMode, size=boxWidth))) + + def onResize(sig, action): + render() + + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + render() + while True: + key = term.inkey(esc_delay=0.05) + keyText = str(key) + if key.name == "KEY_ESCAPE": + return None, None + if key.name == "KEY_ENTER": + value = state["value"] + if not value and requirement["operator"] == ":?": + state["message"] = "%s cannot be empty." % name + render() + continue + _storeValue(envPath, name, value, onValueSaved) + return True, "%s was saved to .env." % name + if key.name == "KEY_BACKSPACE" or keyText in ("\x08", "\x7f"): + state["value"] = state["value"][:-1] + state["message"] = None + render() + elif not key.is_sequence and keyText == "\x15": + state["value"] = "" + state["message"] = None + render() + elif not key.is_sequence and keyText: + printable = "".join( + character + for character in keyText + if character.isprintable() and character not in "\r\n" + ) + if printable: + state["value"] += printable + state["message"] = None + render() + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) + + +def _showPassword(term, renderMode, serviceName, name, value): + safeValue = "".join( + character if character.isprintable() else "?" + for character in value + ) + if not safeValue: + safeValue = "" + + def render(): + boxWidth = _boxWidth(term) + rows = textwrap.wrap( + "%s=%s" % (name, safeValue), + width=max(20, boxWidth - 6), + subsequent_indent=" ", + break_long_words=True, + break_on_hyphens=False, + ) + print(term.clear(), end="") + print(term.black_on_cornsilk4(term.center("%s Password" % serviceName))) + print("") + print(term.center(commonTopBorder(renderMode, size=boxWidth))) + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + for row in rows: + print(term.center(commonTextLine( + renderMode, + row, + size=boxWidth, + paddingBefore=3, + style=term.yellow, + ))) + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + print(term.center(commonTextLine( + renderMode, + "Saved in .env. It can be viewed here again later.", + size=boxWidth, + paddingBefore=3, + ))) + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + print(term.center(commonTextLine( + renderMode, + "[Enter/Esc] Back", + size=boxWidth, + paddingBefore=3, + ))) + print(term.center(commonBottomBorder(renderMode, size=boxWidth))) + + def onResize(sig, action): + render() + + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + render() + while True: + key = term.inkey(esc_delay=0.05) + if key.name in ("KEY_ENTER", "KEY_ESCAPE") or ( + not key.is_sequence and str(key).lower() == "q" + ): + return + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) + + +def _environmentStatus(name, requirement, envPath): + savedEnvironment = loadDotEnv(envPath) + if name in savedEnvironment and ( + requirement["operator"] != ":?" or savedEnvironment[name] != "" + ): + return "saved in .env" + if name in os.environ and ( + requirement["operator"] != ":?" or os.environ[name] != "" + ): + return "set in process environment" + if requirement["operator"] in (":-", "-"): + return "using Compose default" + return "not configured" + + +def _configurePassword( + term, + renderMode, + serviceName, + name, + requirement, + envPath, + onValueSaved, +): + defaultValue = defaultEnvironmentValue(name, requirement) + hasDefault = requirement["operator"] in (":-", "-") or bool(defaultValue) + savedValue = loadDotEnv(envPath).get(name) + defaultLabel = defaultValue if defaultValue else "no password" + entries = [] + if hasDefault: + entries.append(("default", "Use default: %s" % defaultLabel)) + if savedValue is not None: + entries.append(("view", "View password saved in .env")) + entries.extend([ + ("custom", "Enter a custom password"), + ("generate", "Generate a random password and save it"), + (None, "Back to password list"), + ]) + + action = _runMenu( + term, + renderMode, + "%s Password Options" % serviceName, + "Choose how to configure %s:" % name, + entries, + ) + if action is None: + return + if action == "default": + _storeValue(envPath, name, defaultValue, onValueSaved) + _showPassword(term, renderMode, serviceName, name, defaultValue) + return + if action == "view": + _showPassword(term, renderMode, serviceName, name, savedValue) + return + if action == "custom": + success, unusedMessage = _promptForValue( + term, + renderMode, + serviceName, + name, + requirement, + envPath, + onValueSaved, + ) + if success: + _showPassword( + term, + renderMode, + serviceName, + name, + loadDotEnv(envPath)[name], + ) + return + if action == "generate": + value = generateEnvironmentSecret() + _storeValue(envPath, name, value, onValueSaved) + _showPassword(term, renderMode, serviceName, name, value) + + +def _runPasswordOptions( + term, + renderMode, + serviceName, + passwordNames, + requirements, + envPath, + onValueSaved, +): + while True: + entries = [ + ( + name, + "%s [%s]" % ( + name, + _environmentStatus(name, requirements[name], envPath), + ), + ) + for name in passwordNames + ] + entries.append((None, "Back to service settings")) + name = _runMenu( + term, + renderMode, + "%s Password Options" % serviceName, + "Select a password to configure:", + entries, + ) + if name is None: + return + _configurePassword( + term, + renderMode, + serviceName, + name, + requirements[name], + envPath, + onValueSaved, + ) + + +def runEnvironmentOptions( + term, + renderMode, + serviceName, + composeValue, + openServiceOptions=None, + envPath=".env", + onValueSaved=None, +): + requirements = configurableEnvironmentVariables(composeValue) + passwordNames = sorted( + name for name in requirements if isPasswordEnvironmentName(name) + ) + state = { + "selection": 0, + "render": True, + "message": None, + } + + def entries(): + missing = requiredEnvironmentIssues(composeValue, envPath=envPath) + result = [] + if passwordNames: + result.append(( + "Password options (%s)" % len(passwordNames), + lambda: _runPasswordOptions( + term, + renderMode, + serviceName, + passwordNames, + requirements, + envPath, + onValueSaved, + ), + )) + for name in sorted(requirements): + if name in passwordNames: + continue + status = "missing" if "missingEnvironment:%s" % name in missing else "configured" + result.append(( + "Set %s [%s]" % (name, status), + lambda variableName=name: configure(variableName), + )) + if openServiceOptions is not None: + result.append(("Open service-specific options", openServiceOptions)) + result.append(("Back to build menu", None)) + return result + + def configure(name): + success, message = _promptForValue( + term, + renderMode, + serviceName, + name, + requirements[name], + envPath, + onValueSaved, + ) + if success is not None: + state["message"] = message + state["render"] = True + + def render(): + menuEntries = entries() + state["selection"] %= len(menuEntries) + boxWidth = _boxWidth(term) + print(term.clear(), end="") + print(term.black_on_cornsilk4(term.center("%s Service Settings" % serviceName))) + print("") + print(term.center(commonTopBorder(renderMode, size=boxWidth))) + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + print(term.center(commonTextLine( + renderMode, + "Configure values used by this service:", + size=boxWidth, + paddingBefore=4, + ))) + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + for index, (label, unusedAction) in enumerate(menuEntries): + selectedLabel = "-> %s <-" % label if index == state["selection"] else " %s" % label + style = term.blue_on_green if index == state["selection"] else None + print(term.center(commonTextLine( + renderMode, + selectedLabel, + size=boxWidth, + paddingBefore=3, + style=style, + ))) + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + if state["message"]: + print(term.center(commonTextLine( + renderMode, + state["message"], + size=boxWidth, + paddingBefore=4, + style=term.yellow, + ))) + else: + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + print(term.center(commonTextLine( + renderMode, + "[Up/Down] Move [Enter] Select [Esc] Back", + size=boxWidth, + paddingBefore=4, + ))) + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + print(term.center(commonBottomBorder(renderMode, size=boxWidth))) + state["render"] = False + + def onResize(sig, action): + state["render"] = True + render() + + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + with term.fullscreen(): + with term.cbreak(): + while True: + if state["render"]: + render() + key = term.inkey(esc_delay=0.05) + menuEntries = entries() + if key.name == "KEY_ESCAPE" or ( + not key.is_sequence and str(key).lower() == "q" + ): + return True + if key.name == "KEY_UP": + state["selection"] = (state["selection"] - 1) % len(menuEntries) + state["render"] = True + elif key.name == "KEY_DOWN": + state["selection"] = (state["selection"] + 1) % len(menuEntries) + state["render"] = True + elif key.name == "KEY_ENTER": + unusedLabel, action = menuEntries[state["selection"]] + if action is None: + return True + action() + state["render"] = True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) diff --git a/scripts/deps/issue_viewer.py b/scripts/deps/issue_viewer.py new file mode 100644 index 000000000..d6dfe6746 --- /dev/null +++ b/scripts/deps/issue_viewer.py @@ -0,0 +1,128 @@ +import signal +import textwrap + +from deps.chars import ( + commonBottomBorder, + commonEmptyLine, + commonTextLine, + commonTopBorder, +) + + +def issueDisplayRows(issueEntries, contentWidth=78): + rows = [] + for serviceName, issue, issueValue in issueEntries: + if str(issue).startswith("missingEnvironment:"): + prefix = "%s: " % serviceName + else: + prefix = "%s (%s): " % (serviceName, issue) + description = str(issueValue) + wrapped = textwrap.wrap( + prefix + description, + width=max(20, contentWidth), + subsequent_indent=" ", + break_long_words=True, + break_on_hyphens=False, + ) or [prefix.rstrip()] + rows.extend(wrapped) + return rows + + +def compactIssueRows(issueEntries, contentWidth=78, maximumRows=4): + rows = issueDisplayRows(issueEntries, contentWidth=contentWidth) + maximumRows = max(1, maximumRows) + if len(rows) <= maximumRows: + return rows + hiddenRows = len(rows) - maximumRows + 1 + return rows[:maximumRows - 1] + [ + "... %s more lines. Press [I] to view all issues." % hiddenRows + ] + + +def runIssueViewer(term, renderMode, issueEntries): + state = { + "offset": 0, + "render": True, + } + + def dimensions(): + boxWidth = min(80, max(40, term.width - 2)) + visibleRows = max(1, term.height - 8) + rows = issueDisplayRows(issueEntries, contentWidth=boxWidth - 6) + return boxWidth, visibleRows, rows + + def render(): + boxWidth, visibleRows, rows = dimensions() + maximumOffset = max(0, len(rows) - visibleRows) + state["offset"] = max(0, min(state["offset"], maximumOffset)) + visible = rows[state["offset"]:state["offset"] + visibleRows] + firstRow = state["offset"] + 1 if rows else 0 + lastRow = state["offset"] + len(visible) + + print(term.clear(), end="") + print(term.black_on_cornsilk4(term.center("IOTstack Build Issues"))) + print(term.center(commonTopBorder(renderMode, size=boxWidth))) + print(term.center(commonTextLine( + renderMode, + "Issues %s-%s of %s" % (firstRow, lastRow, len(rows)), + size=boxWidth, + paddingBefore=3, + ))) + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + for row in visible: + print(term.center(commonTextLine( + renderMode, + row, + size=boxWidth, + paddingBefore=3, + ))) + for unusedRow in range(visibleRows - len(visible)): + print(term.center(commonEmptyLine(renderMode, size=boxWidth))) + print(term.center(commonTextLine( + renderMode, + "[Up/Down] Scroll [PgUp/PgDn] Page [Esc/Enter] Back", + size=boxWidth, + paddingBefore=3, + ))) + print(term.center(commonBottomBorder(renderMode, size=boxWidth))) + state["render"] = False + + def onResize(sig, action): + state["render"] = True + render() + + originalSignalHandler = signal.getsignal(signal.SIGWINCH) + signal.signal(signal.SIGWINCH, onResize) + try: + with term.fullscreen(): + with term.cbreak(): + while True: + if state["render"]: + render() + key = term.inkey(esc_delay=0.05) + boxWidth, visibleRows, rows = dimensions() + maximumOffset = max(0, len(rows) - visibleRows) + if key.name in ("KEY_ESCAPE", "KEY_ENTER") or ( + not key.is_sequence and str(key).lower() in ("q", "i") + ): + return True + if key.name == "KEY_UP": + state["offset"] = max(0, state["offset"] - 1) + state["render"] = True + elif key.name == "KEY_DOWN": + state["offset"] = min(maximumOffset, state["offset"] + 1) + state["render"] = True + elif key.name == "KEY_PGUP": + state["offset"] = max(0, state["offset"] - visibleRows) + state["render"] = True + elif key.name == "KEY_PGDOWN": + state["offset"] = min(maximumOffset, state["offset"] + visibleRows) + state["render"] = True + elif key.name == "KEY_HOME": + state["offset"] = 0 + state["render"] = True + elif key.name == "KEY_END": + state["offset"] = maximumOffset + state["render"] = True + finally: + signal.signal(signal.SIGWINCH, originalSignalHandler) diff --git a/scripts/deps/menu_renderer.py b/scripts/deps/menu_renderer.py new file mode 100644 index 000000000..8b90aaa20 --- /dev/null +++ b/scripts/deps/menu_renderer.py @@ -0,0 +1,29 @@ +def terminalSupportsMenu(terminalWidth, terminalHeight, minimumWidth=82, minimumHeight=30): + return terminalWidth >= minimumWidth and terminalHeight >= minimumHeight + + +def pageSizeForTerminal(terminalHeight, reservedLines=22): + return max(1, terminalHeight - reservedLines) + + +def issuePanelHeight(issueRows, fixedRows=7): + if issueRows <= 0: + return 0 + return fixedRows + issueRows + + +def serviceOptionsMessage(hasOptions, isSelected): + if not hasOptions: + return "This container has no configurable options." + if not isSelected: + return "Select this container with [Space] before opening its options." + return None + + +def paginationStart(selection, currentStart, pageSize): + pageSize = max(1, pageSize) + if selection >= currentStart + pageSize: + return selection - pageSize + 1 + if selection < currentStart: + return selection + return currentStart diff --git a/scripts/deps/service_hooks.py b/scripts/deps/service_hooks.py new file mode 100644 index 000000000..9c2a72e87 --- /dev/null +++ b/scripts/deps/service_hooks.py @@ -0,0 +1,72 @@ +import hashlib +import importlib.util +import pathlib +import sys + + +HOOK_FUNCTIONS = { + "options": "runOptionsMenu", + "preBuild": "preBuild", + "postBuild": "postBuild", + "runChecks": "runChecks", +} + + +class HookContext: + """State exposed to a service hook. + + New service hooks receive one context object rather than injected globals. + Hooks may update ``services`` in place or replace it with another mapping. + """ + + def __init__(self, services, serviceName, renderMode=None, terminal=None): + self.services = services + self.serviceName = serviceName + self.renderMode = renderMode + self.terminal = terminal + + +def _loadHook(buildScriptPath, serviceName): + sourcePath = pathlib.Path(buildScriptPath).resolve() + digest = hashlib.sha1(str(sourcePath).encode("utf-8")).hexdigest()[:12] + safeServiceName = "".join(character if character.isalnum() else "_" for character in serviceName) + moduleName = "iotstack_service_hook_%s_%s" % (safeServiceName, digest) + spec = importlib.util.spec_from_file_location(moduleName, str(sourcePath)) + if spec is None or spec.loader is None: + raise ImportError("Unable to load service hook '%s'" % sourcePath) + + module = importlib.util.module_from_spec(spec) + sys.modules[moduleName] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(moduleName, None) + raise + + return module + + +def serviceHookAvailable(buildScriptPath, hookName, context): + if hookName not in HOOK_FUNCTIONS: + raise ValueError("Unknown service hook '%s'" % hookName) + + module = _loadHook(buildScriptPath, context.serviceName) + if hookName == "options" and getattr(module, "OPTIONS_AVAILABLE", True) is False: + return False + return callable(getattr(module, HOOK_FUNCTIONS[hookName], None)) + + +def runServiceHook(buildScriptPath, hookName, context): + if hookName not in HOOK_FUNCTIONS: + raise ValueError("Unknown service hook '%s'" % hookName) + + module = _loadHook(buildScriptPath, context.serviceName) + hook = getattr(module, HOOK_FUNCTIONS[hookName], None) + if not callable(hook): + return {} if hookName == "runChecks" else None + result = hook(context) + if hookName == "runChecks" and not isinstance(result, dict): + raise TypeError( + "%s: runChecks(context) must return a dictionary" % buildScriptPath + ) + return result diff --git a/scripts/deps/service_templates.py b/scripts/deps/service_templates.py new file mode 100644 index 000000000..fbdf10714 --- /dev/null +++ b/scripts/deps/service_templates.py @@ -0,0 +1,46 @@ +import os + + +def loadServiceTemplate(yaml, templatesDirectory, serviceName, servicesFileName): + serviceFilePath = os.path.join(templatesDirectory, serviceName, servicesFileName) + with open(serviceFilePath) as yamlServiceFile: + templateServices = yaml.load(yamlServiceFile) + + if not isinstance(templateServices, dict) or serviceName not in templateServices: + raise ValueError("Service template '%s' does not define '%s'" % (serviceFilePath, serviceName)) + + return templateServices + + +def mergeServiceTemplate(dockerComposeServicesYaml, templateServices, reload=False): + for templateServiceName, templateService in templateServices.items(): + if reload or templateServiceName not in dockerComposeServicesYaml: + dockerComposeServicesYaml[templateServiceName] = templateService + + +def removeServiceTemplate(dockerComposeServicesYaml, templateServices): + for templateServiceName in templateServices: + dockerComposeServicesYaml.pop(templateServiceName, None) + + +def restoreSavedServiceTemplates( + yaml, + templatesDirectory, + templateNames, + savedServices, + servicesFileName, +): + if not isinstance(savedServices, dict): + return {} + + restoredServices = {} + for templateName in templateNames: + if templateName not in savedServices: + continue + templateServices = loadServiceTemplate( + yaml, templatesDirectory, templateName, servicesFileName + ) + for serviceName in templateServices: + if serviceName in savedServices: + restoredServices[serviceName] = savedServices[serviceName] + return restoredServices diff --git a/scripts/deps/version_check.py b/scripts/deps/version_check.py index d60740b7b..6d4701dfe 100755 --- a/scripts/deps/version_check.py +++ b/scripts/deps/version_check.py @@ -37,7 +37,7 @@ def checkVersion(requiredVersion, currentVersion): if currentMajor > requiredMajor: return True, '', [] - if currentMajor == requiredMajor and currentMajor > requiredMinor: + if currentMajor == requiredMajor and currentMinor > requiredMinor: return True, '', [] if currentMajor == requiredMajor and currentMinor == requiredMinor and currentBuild >= requiredBuild: diff --git a/scripts/docker_commands.py b/scripts/docker_commands.py index ad7fec1b3..e14c914e4 100755 --- a/scripts/docker_commands.py +++ b/scripts/docker_commands.py @@ -28,75 +28,99 @@ def onResize(sig, action): global currentMenuItemIndex mainRender(1, mainMenuList, currentMenuItemIndex) + def runCommand(command): + exitCode = subprocess.call(command, shell=True) + if exitCode != 0: + print("Command failed with exit status %s." % exitCode) + return False + return True + def startStack(): print("Start Stack:") print("docker-compose up -d --remove-orphans") - subprocess.call("docker-compose up -d", shell=True) + stackStarted = runCommand("docker-compose up -d --remove-orphans") print("") - print("Stack Started") + if stackStarted: + print("Stack Started") + else: + print("Stack was not started") input("Process terminated. Press [Enter] to show menu and continue.") needsRender = 1 - return True + return stackStarted def restartStack(): print("Restarting Stack...") print("Stop Stack:") print("docker-compose down") - subprocess.call("docker-compose down", shell=True) + stackStopped = runCommand("docker-compose down") print("") + if not stackStopped: + print("Stack restart aborted because the stack could not be stopped") + input("Process terminated. Press [Enter] to show menu and continue.") + needsRender = 1 + return False + print("Start Stack:") print("docker-compose up -d --remove-orphans") - subprocess.call("docker-compose up -d", shell=True) + stackStarted = runCommand("docker-compose up -d --remove-orphans") # print("docker-compose restart") # subprocess.call("docker-compose restart", shell=True) print("") - print("Stack Restarted") + if stackStarted: + print("Stack Restarted") + else: + print("Stack was stopped but could not be restarted") input("Process terminated. Press [Enter] to show menu and continue.") needsRender = 1 - return True + return stackStarted def stopStack(): print("Stop Stack:") print("docker-compose down") - subprocess.call("docker-compose down", shell=True) + stackStopped = runCommand("docker-compose down") print("") - print("Stack Stopped") + if stackStopped: + print("Stack Stopped") + else: + print("Stack was not stopped") input("Process terminated. Press [Enter] to show menu and continue.") needsRender = 1 - return True + return stackStopped def stopAllStack(): print("Stop All Stack:") print("docker container stop $(docker container ls -aq)") - subprocess.call("docker container stop $(docker container ls -aq)", shell=True) + allStopped = runCommand("docker container stop $(docker container ls -aq)") print("") input("Process terminated. Press [Enter] to show menu and continue.") needsRender = 1 - return True + return allStopped def pruneVolumes(): print("Stop All Stack:") print("docker container stop $(docker container ls -aq)") - subprocess.call("docker container stop $(docker container ls -aq)", shell=True) + allStopped = runCommand("docker container stop $(docker container ls -aq)") print("") input("Process terminated. Press [Enter] to show menu and continue.") needsRender = 1 - return True + return allStopped def updateAllContainers(): print("Update All Containers:") - print("docker-compose pull") - subprocess.call("docker-compose pull", shell=True) - print("") - print("docker-compose build --no-cache --pull") - subprocess.call("docker-compose build --no-cache --pull", shell=True) - print("") - print("docker-compose up -d") - subprocess.call("docker-compose up -d", shell=True) - print("") - print("docker system prune -f") - subprocess.call("docker system prune -f", shell=True) - print("") + commands = [ + "docker-compose pull", + "docker-compose build --no-cache --pull", + "docker-compose up -d", + "docker system prune -f", + ] + for command in commands: + print(command) + if not runCommand(command): + print("Container update aborted after a command failed.") + input("Process terminated. Press [Enter] to show menu and continue.") + needsRender = 1 + return False + print("") input("Process terminated. Press [Enter] to show menu and continue.") needsRender = 1 return True @@ -104,20 +128,20 @@ def updateAllContainers(): def deleteAndPruneVolumes(): print("Delete and prune volumes:") print("docker system prune --volumes") - subprocess.call("docker system prune --volumes", shell=True) + volumesPruned = runCommand("docker system prune --volumes") print("") input("Process terminated. Press [Enter] to show menu and continue.") needsRender = 1 - return True + return volumesPruned def deleteAndPruneImages(): print("Delete and prune volumes:") print("docker image prune -a") - subprocess.call("docker image prune -a", shell=True) + imagesPruned = runCommand("docker image prune -a") print("") input("Process terminated. Press [Enter] to show menu and continue.") needsRender = 1 - return True + return imagesPruned def monitorLogs(): print("Monitor Logs:") @@ -163,7 +187,7 @@ def goBack(): needsRender = 1 def renderHotZone(term, menu, selection, hotzoneLocation): - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") lineLengthAtTextStart = 71 for (index, menuItem) in enumerate(menu): @@ -244,6 +268,7 @@ def isMenuItemSelectable(menu, index): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, mainMenuList, currentMenuItemIndex) + needsRender = 0 dockerCommandsSelectionInProgress = True with term.cbreak(): while dockerCommandsSelectionInProgress: @@ -289,6 +314,6 @@ def isMenuItemSelectable(menu, index): return True -originalSignalHandler = signal.getsignal(signal.SIGINT) +originalSignalHandler = signal.getsignal(signal.SIGWINCH) main() signal.signal(signal.SIGWINCH, originalSignalHandler) \ No newline at end of file diff --git a/scripts/menu_main.py b/scripts/menu_main.py index b4dfc17ba..4e61b87ea 100755 --- a/scripts/menu_main.py +++ b/scripts/menu_main.py @@ -83,7 +83,9 @@ def onResize(sig, action): global mainMenuList global currentMenuItemIndex global screenActive + global hotzoneLocation if screenActive: + hotzoneLocation = [((term.height // 16) + 6), 0] mainRender(1, mainMenuList, currentMenuItemIndex) # Menu Functions @@ -114,23 +116,30 @@ def buildStack(): needsRender = 1 def runExampleMenu(): - exampleMenuFilePath = "./.templates/example_template/example_build.py" - with open(exampleMenuFilePath, "rb") as pythonDynamicImportFile: - code = compile(pythonDynamicImportFile.read(), exampleMenuFilePath, "exec") - # execGlobals = globals() - execGlobals = { - "renderMode": renderMode - } - execLocals = locals() - execGlobals["currentServiceName"] = 'SERVICENAME' - execGlobals["toRun"] = 'runOptionsMenu' + global screenActive + from deps.service_hooks import HookContext, runServiceHook + import ruamel.yaml + + exampleMenuFilePath = "./.templates/example_template/build.py" + exampleServiceFilePath = "./.templates/example_template/example_service.yml" + yaml = ruamel.yaml.YAML() + with open(exampleServiceFilePath) as exampleServiceFile: + services = yaml.load(exampleServiceFile) + serviceName = next(iter(services)) + context = HookContext( + services=services, + serviceName=serviceName, + renderMode=renderMode, + terminal=term, + ) screenActive = False - exec(code, execGlobals, execLocals) + runServiceHook(exampleMenuFilePath, "options", context) signal.signal(signal.SIGWINCH, onResize) screenActive = True def dockerCommands(): global needsRender + global screenActive dockerCommandsFilePath = "./scripts/docker_commands.py" with open(dockerCommandsFilePath, "rb") as pythonDynamicImportFile: code = compile(pythonDynamicImportFile.read(), dockerCommandsFilePath, "exec") @@ -148,6 +157,7 @@ def dockerCommands(): def miscCommands(): global needsRender + global screenActive dockerCommandsFilePath = "./scripts/misc_commands.py" with open(dockerCommandsFilePath, "rb") as pythonDynamicImportFile: code = compile(pythonDynamicImportFile.read(), dockerCommandsFilePath, "exec") @@ -209,8 +219,8 @@ def skipItem(currentMenuItemIndex, direction): return currentMenuItemIndex def deletePromptFiles(): - # global promptFiles - # global currentMenuItemIndex + global promptFiles + global currentMenuItemIndex if os.path.exists(".project_outofdate"): os.remove(".project_outofdate") if os.path.exists(".docker_outofdate"): @@ -327,12 +337,17 @@ def addPotentialMenuItem(menuItemName, hasSpacer=True): return False def removeMenuItemByLabel(potentialItemKey): + global currentMenuItemIndex i = -1 for menuItem in mainMenuList: i += 1 if menuItem[0] == potentialMenu[potentialItemKey]["menuItem"][0]: potentialMenu[potentialItemKey]["added"] = False mainMenuList.pop(i) + if len(mainMenuList) > 0: + currentMenuItemIndex = currentMenuItemIndex % len(mainMenuList) + return True + return False def doPotentialMenuCheck(projectStatus, dockerVersion=True, promptFiles=False): global needsRender @@ -341,7 +356,8 @@ def doPotentialMenuCheck(projectStatus, dockerVersion=True, promptFiles=False): addPotentialMenuItem("deletePromptFiles") needsRender = 2 else: - removeMenuItemByLabel("deletePromptFiles") + if removeMenuItemByLabel("deletePromptFiles"): + needsRender = 1 # if (projectStatus.poll() == None): # addPotentialMenuItem("updatesCheck", False) @@ -376,12 +392,12 @@ def checkIfPromptFilesExist(): return False def renderHotZone(term, menu, selection): - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): if index == selection: - print(term.center('-> {t.blue_on_green}{title}{t.normal} <-'.format(t=term, title=menuItem[0]))) + print(term.clear_eol + term.center('-> {t.blue_on_green}{title}{t.normal} <-'.format(t=term, title=menuItem[0]))) else: - print(term.center('{title}'.format(t=term, title=menuItem[0]))) + print(term.clear_eol + term.center('{title}'.format(t=term, title=menuItem[0]))) def mainRender(needsRender, menu, selection): term = Terminal() @@ -432,6 +448,7 @@ def isMenuItemSelectable(menu, index): with term.fullscreen(): checkRenderOptions() mainRender(needsRender, mainMenuList, currentMenuItemIndex) # Initial Draw + needsRender = 0 with term.cbreak(): while selectionInProgress: menuNavigateDirection = 0 diff --git a/scripts/misc_commands.py b/scripts/misc_commands.py index 8c311c5d6..a54743460 100755 --- a/scripts/misc_commands.py +++ b/scripts/misc_commands.py @@ -91,7 +91,7 @@ def onResize(sig, action): def renderHotZone(term, menu, selection, hotzoneLocation): lineLengthAtTextStart = 71 - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") for (index, menuItem) in enumerate(menu): toPrint = "" if index == selection: @@ -166,6 +166,7 @@ def isMenuItemSelectable(menu, index): with term.fullscreen(): menuNavigateDirection = 0 mainRender(needsRender, mainMenuList, currentMenuItemIndex) + needsRender = 0 dockerCommandsSelectionInProgress = True with term.cbreak(): while dockerCommandsSelectionInProgress: @@ -210,6 +211,6 @@ def isMenuItemSelectable(menu, index): return True -originalSignalHandler = signal.getsignal(signal.SIGINT) +originalSignalHandler = signal.getsignal(signal.SIGWINCH) main() signal.signal(signal.SIGWINCH, originalSignalHandler) diff --git a/scripts/native_installs.py b/scripts/native_installs.py index a55ce9bfe..301aad8b4 100755 --- a/scripts/native_installs.py +++ b/scripts/native_installs.py @@ -96,7 +96,7 @@ def goBack(): needsRender = 1 def renderHotZone(term, menu, selection, hotzoneLocation): - print(term.move(hotzoneLocation[0], hotzoneLocation[1])) + print(term.move(hotzoneLocation[0], hotzoneLocation[1]), end="") lineLengthAtTextStart = 71 for (index, menuItem) in enumerate(menu): @@ -178,6 +178,7 @@ def isMenuItemSelectable(menu, index): signal.signal(signal.SIGWINCH, onResize) menuNavigateDirection = 0 mainRender(needsRender, mainMenuList, currentMenuItemIndex) + needsRender = 0 dockerCommandsSelectionInProgress = True with term.cbreak(): while dockerCommandsSelectionInProgress: @@ -227,5 +228,7 @@ def isMenuItemSelectable(menu, index): screenActive = False return True +originalSignalHandler = signal.getsignal(signal.SIGWINCH) main() +signal.signal(signal.SIGWINCH, originalSignalHandler) diff --git a/scripts/restore.sh b/scripts/restore.sh index 02c9e8a9c..4818b9ea8 100755 --- a/scripts/restore.sh +++ b/scripts/restore.sh @@ -14,17 +14,19 @@ # Will restore from the backup file "./backups/some_other_backup.tar.gz" and will not warn that data will be deleted. # -if [ -d "./menu.sh" ]; then +if [ ! -f "./menu.sh" ]; then echo "./menu.sh file was not found. Ensure that you are running this from IOTstack's directory." exit 1 fi -echo "Restoring from a backup will erase all existing data." -read -p "Continue [y/N]? " -n 1 -r PROCEED_WITH_RESTORE -echo "" -if [[ ! $PROCEED_WITH_RESTORE =~ ^[Yy]$ ]]; then - echo "Restore Cancelled." - exit 0 +if [ "${2:-}" != "noask" ]; then + echo "Restoring from a backup will erase all existing data." + read -p "Continue [y/N]? " -n 1 -r PROCEED_WITH_RESTORE + echo "" + if [[ ! $PROCEED_WITH_RESTORE =~ ^[Yy]$ ]]; then + echo "Restore Cancelled." + exit 0 + fi fi RESTOREFILENAME="backup.tar.gz" @@ -40,8 +42,7 @@ BACKUPFILE="$BASEDIR/backup/backup_$BASERESTOREFILE.tar.gz" [ -d ./backups ] || mkdir -p ./backups [ -d ./backups/logs ] || mkdir -p ./backups/logs -[ -d ./.tmp ] || sudo rm -rf ./.tmp -[ -d ./tmp ] || mkdir -p ./tmp +[ -d ./.tmp ] || mkdir -p ./.tmp touch $LOGFILE echo "" > $LOGFILE @@ -59,27 +60,37 @@ if [ ! -f $RESTOREFILE ]; then echo "### End of log ###" >> $LOGFILE exit 2 fi +if ! tar -tzf "$RESTOREFILE" > /dev/null 2>> "$LOGFILE"; then + echo "Backup archive '$RESTOREFILE' is unreadable or corrupt. Cancelling restore." >> "$LOGFILE" + cat "$LOGFILE" + exit 3 +fi + # Remove old files and folders sudo rm -rf ./services/ >> $LOGFILE 2>&1 sudo rm -rf ./volumes/ >> $LOGFILE 2>&1 sudo rm -rf ./compose-override.yml >> $LOGFILE 2>&1 sudo rm -rf ./docker-compose.yml >> $LOGFILE 2>&1 +sudo rm -rf ./.env >> $LOGFILE 2>&1 sudo rm -rf ./extra/ >> $LOGFILE 2>&1 sudo rm -rf ./postbuild.sh >> $LOGFILE 2>&1 sudo rm -rf ./pre_backup.sh >> $LOGFILE 2>&1 sudo rm -rf ./post_backup.sh >> $LOGFILE 2>&1 sudo rm -rf ./post_restore.sh >> $LOGFILE 2>&1 -sudo rm -rf ./post_restore.sh >> $LOGFILE 2>&1 +sudo rm -rf ./docker-compose.override.yml >> $LOGFILE 2>&1 -sudo tar -zxvf \ - $RESTOREFILE >> $LOGFILE 2>&1 +if ! sudo tar -zxvf "$RESTOREFILE" >> "$LOGFILE" 2>&1; then + echo "Backup extraction failed." >> "$LOGFILE" + cat "$LOGFILE" + exit 4 +fi echo "" >> $LOGFILE echo "Executing post restore scripts" >> $LOGFILE bash ./scripts/backup_restore/post_restore_complete.sh >> $LOGFILE 2>&1 -echo "" > $LOGFILE +echo "" >> $LOGFILE echo "Finished At: $(date +"%Y-%m-%dT%H-%M-%S")" >> $LOGFILE echo "" >> $LOGFILE diff --git a/scripts/yaml_merge.py b/scripts/yaml_merge.py index ad7477ac0..cbd9813d8 100755 --- a/scripts/yaml_merge.py +++ b/scripts/yaml_merge.py @@ -5,7 +5,7 @@ yaml = ruamel.yaml.YAML() yaml.preserve_quotes = True -if sys.argv[1] == "--pyyaml-version": +if len(sys.argv) > 1 and sys.argv[1] == "--pyyaml-version": try: print("pyyaml", yaml.__version__) sys.exit(0) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/fixtures/legacy_build.py b/tests/fixtures/legacy_build.py new file mode 100644 index 000000000..643a0ccbc --- /dev/null +++ b/tests/fixtures/legacy_build.py @@ -0,0 +1,2 @@ +def runChecks(context): + return {} diff --git a/tests/test_regressions.py b/tests/test_regressions.py new file mode 100644 index 000000000..0c074c285 --- /dev/null +++ b/tests/test_regressions.py @@ -0,0 +1,729 @@ +import os +import pathlib +import shutil +import subprocess +import sys +import tempfile +import unittest + +from ruamel.yaml import YAML + + +ROOT = pathlib.Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +from deps.chars import commonTextLine +from deps.compose_environment import ( + requiredEnvironmentIssues, + configurableEnvironmentVariables, + defaultEnvironmentValue, + isPasswordEnvironmentName, + loadDotEnv, + requiredEnvironmentVariables, + restoreEnvironmentVariableReferences, + setDotEnvValue, +) +from deps.menu_renderer import issuePanelHeight, pageSizeForTerminal, paginationStart, serviceOptionsMessage, terminalSupportsMenu +from deps.issue_viewer import compactIssueRows, issueDisplayRows +from deps.environment_options import generateEnvironmentSecret +from deps.service_hooks import ( + HookContext, + runServiceHook, + serviceHookAvailable, +) +from deps.service_templates import loadServiceTemplate, mergeServiceTemplate, removeServiceTemplate, restoreSavedServiceTemplates +from deps.version_check import checkVersion +import menu_main + + +class VersionCheckTests(unittest.TestCase): + def test_rejects_older_minor_version(self): + self.assertFalse(checkVersion("18.2.0", "18.1.99")[0]) + + def test_accepts_newer_minor_version(self): + self.assertTrue(checkVersion("18.2.0", "18.3.0")[0]) + + def test_rejects_older_build_version(self): + self.assertFalse(checkVersion("18.2.5", "18.2.4")[0]) + + def test_shell_version_check_rejects_older_minor_version(self): + source = (ROOT / "menu.sh").read_text() + functionSource = source[source.index("function minimum_version_check"):source.index("function check_git_updates")] + result = subprocess.run( + ["bash", "-c", functionSource + "\nminimum_version_check 3.6.9 3 5 99"], + capture_output=True, + text=True, + ) + self.assertNotEqual(0, result.returncode) + self.assertEqual("false", result.stdout.strip()) + +class ComposeEnvironmentTests(unittest.TestCase): + def test_required_variable_has_a_short_actionable_warning(self): + issues = requiredEnvironmentIssues( + {"environment": ["PASSWORD=${PASSWORD:?add PASSWORD to .env}"]}, + envPath="/missing/iotstack-test.env", + processEnvironment={}, + ) + self.assertEqual( + "PASSWORD is required. Open Options to configure it.", + issues["missingEnvironment:PASSWORD"], + ) + + def test_dotenv_and_process_environment_satisfy_requirements(self): + with tempfile.TemporaryDirectory() as temporaryDirectory: + envPath = pathlib.Path(temporaryDirectory) / ".env" + envPath.write_text("FROM_FILE=secret\nEMPTY=\n") + values = { + "file": "${FROM_FILE:?missing}", + "process": "${FROM_PROCESS:?missing}", + "emptyAllowed": "${EMPTY?missing}", + "emptyRejected": "${EMPTY:?missing}", + } + issues = requiredEnvironmentIssues( + values, + envPath=envPath, + processEnvironment={"FROM_PROCESS": "secret"}, + ) + self.assertEqual(["missingEnvironment:EMPTY"], list(issues)) + + def test_gitea_reports_each_missing_required_password(self): + services = YAML().load( + (ROOT / ".templates/gitea/service.yml").read_text() + ) + issues = requiredEnvironmentIssues( + services, + envPath="/missing/iotstack-test.env", + processEnvironment={}, + ) + self.assertIn("missingEnvironment:GITEA_DB_PASSWORD", issues) + self.assertIn("missingEnvironment:GITEA_DB_ROOT_PASSWORD", issues) + + def test_configured_gitea_settings_remain_discoverable(self): + services = YAML().load( + (ROOT / ".templates/gitea/service.yml").read_text() + ) + with tempfile.TemporaryDirectory(dir="/tmp") as temporaryDirectory: + envPath = pathlib.Path(temporaryDirectory) / ".env" + envPath.write_text( + "GITEA_DB_PASSWORD=user\n" + "GITEA_DB_ROOT_PASSWORD=root\n" + "GITEA_SECRET_KEY=secret\n" + "GITEA_INTERNAL_TOKEN=token\n" + ) + variables = configurableEnvironmentVariables(services) + issues = requiredEnvironmentIssues( + services, + envPath=envPath, + processEnvironment={}, + ) + self.assertEqual( + { + "GITEA_DB_PASSWORD", + "GITEA_DB_ROOT_PASSWORD", + "GITEA_INTERNAL_TOKEN", + "GITEA_SECRET_KEY", + }, + set(variables), + ) + self.assertEqual({}, issues) + + def test_saved_gitea_values_can_rejoin_dotenv_interpolation(self): + template = YAML().load( + (ROOT / ".templates/gitea/service.yml").read_text() + ) + current = YAML().load( + (ROOT / ".templates/gitea/service.yml").read_text() + ) + current["gitea"]["ports"][0] = "8123:3000/tcp" + current["gitea"]["environment"] = [ + value.replace( + "GITEA__database__PASSWD=${GITEA_DB_PASSWORD:?eg echo GITEA_DB_PASSWORD=userPassword >>~/IOTstack/.env}", + "GITEA__database__PASSWD=old-user", + ) + for value in current["gitea"]["environment"] + ] + current["gitea_db"]["environment"] = [ + value.replace("${GITEA_DB_PASSWORD:?eg echo GITEA_DB_PASSWORD=userPassword >>~/IOTstack/.env}", "old-user") + for value in current["gitea_db"]["environment"] + ] + + changed = restoreEnvironmentVariableReferences( + current, template, "GITEA_DB_PASSWORD" + ) + + self.assertEqual(2, changed) + self.assertEqual("8123:3000/tcp", current["gitea"]["ports"][0]) + self.assertIn("${GITEA_DB_PASSWORD:?", str(current)) + + def test_required_settings_can_be_written_without_losing_existing_env(self): + with tempfile.TemporaryDirectory(dir="/tmp") as temporaryDirectory: + envPath = pathlib.Path(temporaryDirectory) / ".env" + envPath.write_text("# keep this comment\nUNCHANGED=yes\nPASSWORD=old\n") + setDotEnvValue(envPath, "PASSWORD", "new-secret") + setDotEnvValue(envPath, "DEVICE_PATH", "/dev/ttyUSB0") + contents = envPath.read_text() + protectedEnvPath = pathlib.Path(temporaryDirectory) / "new.env" + setDotEnvValue(protectedEnvPath, "PASSWORD", "secret") + protectedMode = protectedEnvPath.stat().st_mode & 0o777 + self.assertEqual(0o600, protectedMode) + self.assertIn("# keep this comment", contents) + self.assertIn("UNCHANGED=yes", contents) + self.assertEqual(1, contents.count("PASSWORD=")) + self.assertIn("PASSWORD=new-secret", contents) + self.assertIn("DEVICE_PATH=/dev/ttyUSB0", contents) + + def test_dotenv_complex_values_round_trip_exactly(self): + values = [ + "space # value", + 'quote"value', + "slash\\value", + "dollar$value", + "two$$dollars", + ] + with tempfile.TemporaryDirectory(dir="/tmp") as temporaryDirectory: + envPath = pathlib.Path(temporaryDirectory) / ".env" + for index, value in enumerate(values): + setDotEnvValue(envPath, "VALUE_%s" % index, value) + loaded = loadDotEnv(envPath) + self.assertEqual(values, [loaded["VALUE_%s" % index] for index in range(len(values))]) + + def test_all_bundled_required_setting_types_are_discovered(self): + discovered = {} + for serviceFile in (ROOT / ".templates").glob("*/service.yml"): + requirements = requiredEnvironmentVariables(YAML().load(serviceFile.read_text())) + if requirements: + discovered[serviceFile.parent.name] = set(requirements) + self.assertIn("GITEA_DB_PASSWORD", discovered["gitea"]) + self.assertIn("WORDPRESS_HOSTNAME", discovered["wordpress"]) + self.assertIn("DUCKDNS_TOKEN", discovered["duckdns"]) + self.assertIn("ZIGBEE2MQTT_DEVICE_PATH", discovered["zigbee2mqtt"]) + self.assertGreaterEqual(len(discovered), 13) + + def test_generated_passwords_use_random_alphanumeric_values(self): + first = generateEnvironmentSecret() + second = generateEnvironmentSecret() + self.assertEqual(32, len(first)) + self.assertTrue(first.isalnum()) + self.assertNotEqual(first, second) + + def test_optional_passwords_and_sensitive_values_are_configurable(self): + expected = { + "deconz": "DECONZ_VNC_PASSWORD", + "gitea": "GITEA_SECRET_KEY", + "influxdb2": "INFLUXDB2_ADMIN_TOKEN", + "mjpg-streamer": "MJPG_STREAMER_PASSWORD", + "pihole": "PIHOLE_ADMIN_PASSWORD", + "pihole6": "PIHOLE_ADMIN_PASSWORD", + } + for serviceName, variableName in expected.items(): + services = YAML().load( + (ROOT / ".templates" / serviceName / "service.yml").read_text() + ) + self.assertIn(variableName, configurableEnvironmentVariables(services)) + + def test_database_passwords_have_documented_defaults(self): + expected = { + "MARIADB_ROOT_PASSWORD": "IOtSt4ckToorMariaDb", + "MARIADB_USER_PASSWORD": "IOtSt4ckmariaDbPw", + "NEXTCLOUD_DB_ROOT_PASSWORD": "IOtSt4ckToorMySqlDb", + "NEXTCLOUD_DB_USER_PASSWORD": "IOtSt4ckmySqlDbPw", + } + variables = {} + for serviceName in ("mariadb", "nextcloud"): + services = YAML().load( + (ROOT / ".templates" / serviceName / "service.yml").read_text() + ) + variables.update(configurableEnvironmentVariables(services)) + for name, expectedDefault in expected.items(): + self.assertEqual( + expectedDefault, + defaultEnvironmentValue(name, variables[name]), + ) + + def test_required_passwords_all_offer_documented_defaults(self): + missingDefaults = [] + for serviceFile in (ROOT / ".templates").glob("*/service.yml"): + requirements = requiredEnvironmentVariables( + YAML().load(serviceFile.read_text()) + ) + for name, requirement in requirements.items(): + if isPasswordEnvironmentName(name) and not defaultEnvironmentValue( + name, requirement + ): + missingDefaults.append("%s:%s" % (serviceFile.parent.name, name)) + self.assertEqual([], missingDefaults) + + def test_mjpg_streamer_credentials_are_stable_and_configurable(self): + services = YAML().load( + (ROOT / ".templates/mjpg-streamer/service.yml").read_text() + ) + variables = configurableEnvironmentVariables(services) + self.assertEqual( + "IOtSt4ckMJPG", + defaultEnvironmentValue("MJPG_STREAMER_PASSWORD", variables["MJPG_STREAMER_PASSWORD"]), + ) + self.assertIn("${MJPG_STREAMER_USERNAME:-iotstack}", str(services)) + + def test_required_interpolation_wins_over_an_optional_duplicate(self): + variables = configurableEnvironmentVariables([ + "${PASSWORD:-default}", + "${PASSWORD?required}", + ]) + self.assertEqual("?", variables["PASSWORD"]["operator"]) +class PaginationTests(unittest.TestCase): + def test_page_boundary_scrolls_one_row(self): + self.assertEqual(1, paginationStart(selection=10, currentStart=0, pageSize=10)) + + def test_scrolling_up_places_selection_at_start(self): + self.assertEqual(3, paginationStart(selection=3, currentStart=5, pageSize=10)) + + def test_small_terminal_never_has_negative_page_size(self): + self.assertEqual(1, pageSizeForTerminal(terminalHeight=10)) + + def test_page_automatically_fills_available_terminal_height(self): + self.assertEqual(13, pageSizeForTerminal(terminalHeight=40, reservedLines=27)) + + def test_issue_panel_height_includes_frame_rows(self): + self.assertEqual(0, issuePanelHeight(0)) + self.assertEqual(9, issuePanelHeight(2)) + + def test_service_options_feedback_matches_service_state(self): + self.assertEqual( + "This container has no configurable options.", + serviceOptionsMessage(False, False), + ) + self.assertIn("Select this container", serviceOptionsMessage(True, False)) + self.assertIsNone(serviceOptionsMessage(True, True)) + + def test_long_issues_wrap_instead_of_truncating(self): + rows = issueDisplayRows([ + ( + "gitea", + "missingEnvironment:GITEA_DB_PASSWORD", + "A required setting with a description that needs another line.", + ), + ], contentWidth=32) + self.assertGreater(len(rows), 1) + self.assertTrue(all(len(row) <= 32 for row in rows)) + + def test_compact_issue_rows_link_to_scrolling_viewer(self): + rows = compactIssueRows([ + ("one", "first", "First issue"), + ("two", "second", "Second issue"), + ("three", "third", "Third issue"), + ], contentWidth=40, maximumRows=2) + self.assertEqual(2, len(rows)) + self.assertIn("Press [I]", rows[-1]) + + def test_build_menu_uses_fallback_for_narrow_terminal(self): + self.assertFalse(terminalSupportsMenu(terminalWidth=80, terminalHeight=24)) + + def test_build_menu_supports_minimum_terminal_dimensions(self): + self.assertTrue(terminalSupportsMenu(terminalWidth=82, terminalHeight=30)) + + +class MenuRenderingTests(unittest.TestCase): + def test_text_line_matches_standard_border_width(self): + line = commonTextLine("ascii", "Warning", paddingBefore=6) + self.assertEqual(82, len(line)) + self.assertEqual("| Warning", line[:14]) + self.assertTrue(line.endswith("|")) + + def test_text_line_styling_does_not_affect_padding(self): + style = lambda text: "%s" % text + plain = commonTextLine("ascii", "Warning", paddingBefore=6) + styled = commonTextLine("ascii", "Warning", paddingBefore=6, style=style) + self.assertEqual(plain.count(" "), styled.count(" ")) + self.assertIn("Warning", styled) + + +class ServiceTemplateTests(unittest.TestCase): + def setUp(self): + self.yaml = YAML() + self.templatesDirectory = str(ROOT / ".templates") + + def test_gitea_companion_database_is_loaded(self): + template = loadServiceTemplate(self.yaml, self.templatesDirectory, "gitea", "service.yml") + selected = {"gitea": {"custom": "preserved"}} + mergeServiceTemplate(selected, template) + self.assertEqual({"custom": "preserved"}, selected["gitea"]) + self.assertIn("gitea_db", selected) + + def test_wordpress_companion_database_is_loaded_and_removed(self): + template = loadServiceTemplate(self.yaml, self.templatesDirectory, "wordpress", "service.yml") + selected = {} + mergeServiceTemplate(selected, template) + self.assertEqual({"wordpress", "wordpress_db"}, set(selected)) + removeServiceTemplate(selected, template) + self.assertEqual({}, selected) + + + def test_saved_companion_service_configuration_is_restored(self): + savedServices = { + "gitea": {"marker": "saved root"}, + "gitea_db": {"marker": "saved database"}, + "orphan": {"marker": "not owned"}, + } + restored = restoreSavedServiceTemplates( + self.yaml, + self.templatesDirectory, + ["gitea", "grafana"], + savedServices, + "service.yml", + ) + self.assertEqual({"gitea", "gitea_db"}, set(restored)) + self.assertEqual("saved database", restored["gitea_db"]["marker"]) + + def test_every_template_defines_its_directory_service(self): + for serviceFile in (ROOT / ".templates").glob("*/service.yml"): + serviceName = serviceFile.parent.name + template = loadServiceTemplate(self.yaml, self.templatesDirectory, serviceName, "service.yml") + self.assertIn(serviceName, template) + + def test_openhab_environment_uses_valid_mapping_form(self): + template = loadServiceTemplate(self.yaml, self.templatesDirectory, "openhab", "service.yml") + self.assertIsInstance(template["openhab"]["environment"], dict) + + +class ServiceHookTests(unittest.TestCase): + def setUp(self): + self.yaml = YAML() + + def test_example_uses_modern_hook_api(self): + buildScript = ROOT / ".templates/example_template/build.py" + serviceFile = ROOT / ".templates/example_template/example_service.yml" + services = self.yaml.load(serviceFile.read_text()) + serviceName = next(iter(services)) + context = HookContext(services, serviceName, renderMode="ascii") + + self.assertTrue(serviceHookAvailable(buildScript, "options", context)) + self.assertTrue(serviceHookAvailable(buildScript, "runChecks", context)) + self.assertEqual({}, runServiceHook(buildScript, "runChecks", context)) + + def test_converted_bundled_hook_uses_modern_api(self): + buildScript = ROOT / ".templates/openhab/build.py" + services = self.yaml.load((ROOT / ".templates/openhab/service.yml").read_text()) + context = HookContext(services, "openhab", renderMode="ascii") + + self.assertTrue(serviceHookAvailable(buildScript, "runChecks", context)) + self.assertEqual({}, runServiceHook(buildScript, "runChecks", context)) + + def test_esphome_checks_do_not_generate_credentials(self): + buildScript = ROOT / ".templates/esphome/build.py" + services = self.yaml.load( + (ROOT / ".templates/esphome/service.yml").read_text() + ) + context = HookContext(services, "esphome", renderMode="ascii") + with tempfile.TemporaryDirectory() as temporaryDirectory: + previousDirectory = os.getcwd() + try: + os.chdir(temporaryDirectory) + result = runServiceHook(buildScript, "runChecks", context) + envCreated = pathlib.Path(".env").exists() + finally: + os.chdir(previousDirectory) + self.assertEqual({}, result) + self.assertFalse(envCreated) + self.assertNotIn("generateRandomString", buildScript.read_text()) + + def test_empty_custom_option_menus_are_not_advertised(self): + for serviceName in ("dozzle", "home_assistant", "influxdb", "mariadb"): + buildScript = ROOT / ".templates" / serviceName / "build.py" + services = self.yaml.load( + (ROOT / ".templates" / serviceName / "service.yml").read_text() + ) + context = HookContext(services, serviceName, renderMode="ascii") + self.assertFalse( + serviceHookAvailable(buildScript, "options", context), + serviceName, + ) + + def test_prebuild_keeps_current_service_settings(self): + for serviceName in ("influxdb", "mariadb"): + with tempfile.TemporaryDirectory() as temporaryDirectory: + temporaryRoot = pathlib.Path(temporaryDirectory) + (temporaryRoot / "services").mkdir() + services = self.yaml.load( + (ROOT / ".templates" / serviceName / "service.yml").read_text() + ) + services[serviceName]["x-iotstack-test-marker"] = "current selection" + context = HookContext(services, serviceName, renderMode="ascii") + previousDirectory = os.getcwd() + try: + os.chdir(temporaryRoot) + result = runServiceHook( + ROOT / ".templates" / serviceName / "build.py", + "preBuild", + context, + ) + finally: + os.chdir(previousDirectory) + self.assertTrue(result) + self.assertEqual( + "current selection", + context.services[serviceName]["x-iotstack-test-marker"], + ) + + def test_hardware_prebuild_fails_cleanly_without_configuration(self): + for serviceName in ("deconz", "otbr"): + with tempfile.TemporaryDirectory() as temporaryDirectory: + temporaryRoot = pathlib.Path(temporaryDirectory) + shutil.copytree( + ROOT / ".templates" / serviceName, + temporaryRoot / ".templates" / serviceName, + ) + (temporaryRoot / "services").mkdir() + services = self.yaml.load( + (temporaryRoot / ".templates" / serviceName / "service.yml").read_text() + ) + context = HookContext(services, serviceName, renderMode="ascii") + previousDirectory = os.getcwd() + try: + os.chdir(temporaryRoot) + result = runServiceHook( + ROOT / ".templates" / serviceName / "build.py", + "preBuild", + context, + ) + finally: + os.chdir(previousDirectory) + self.assertFalse(result) + + def test_hooks_do_not_require_version_marker(self): + buildScript = ROOT / "tests/fixtures/legacy_build.py" + context = HookContext({}, "legacy-service", renderMode="ascii") + + self.assertTrue(serviceHookAvailable(buildScript, "runChecks", context)) + self.assertEqual({}, runServiceHook(buildScript, "runChecks", context)) + + def test_every_bundled_hook_omits_version_marker(self): + for buildScript in (ROOT / ".templates").glob("*/build.py"): + self.assertNotIn("HOOK_API_VERSION", buildScript.read_text(), str(buildScript)) + + def test_every_bundled_hook_can_be_inspected(self): + servicesWithOptions = { + "adminer", "deconz", "diyhue", "gitea", "grafana", + "motioneye", "n8n", + "nextcloud", "nodered", "otbr", "portainer-ce", + "python-matter-server", "transmission", + } + for buildScript in (ROOT / ".templates").glob("*/build.py"): + serviceName = buildScript.parent.name + if serviceName == "example_template": + continue + serviceFile = buildScript.parent / "service.yml" + services = self.yaml.load(serviceFile.read_text()) + context = HookContext(services, serviceName, renderMode="ascii") + + self.assertTrue(serviceHookAvailable(buildScript, "runChecks", context)) + self.assertTrue(serviceHookAvailable(buildScript, "preBuild", context)) + self.assertTrue(serviceHookAvailable(buildScript, "postBuild", context)) + self.assertEqual( + serviceName in servicesWithOptions, + serviceHookAvailable(buildScript, "options", context), + ) + + def test_hook_path_contains_no_dynamic_execution(self): + self.assertNotIn("exec(", (ROOT / "scripts/buildstack_menu.py").read_text()) + self.assertNotIn("exec(", (ROOT / "scripts/deps/service_hooks.py").read_text()) + for buildScript in (ROOT / ".templates").glob("*/build.py"): + self.assertNotIn("eval(toRun)", buildScript.read_text(), str(buildScript)) + + +class SourceRegressionTests(unittest.TestCase): + def test_legacy_password_cache_cannot_override_environment_settings(self): + for relativePath in ( + ".templates/deconz/build.py", ".templates/influxdb/build.py", + ".templates/mariadb/build.py", ".templates/nextcloud/build.py", + ): + source = (ROOT / relativePath).read_text() + self.assertNotIn("buildCacheServices", source) + self.assertNotIn("Password randomisation", source) + self.assertNotIn("generateRandomString", source) + + def test_backup_manifest_includes_both_override_files(self): + source = (ROOT / "scripts/backup.sh").read_text() + self.assertIn('echo "./docker-compose.override.yml" >> $BACKUPLIST', source) + self.assertIn('echo "./compose-override.yml" >> $BACKUPLIST', source) + self.assertIn('echo "./.env" >> $BACKUPLIST', source) + self.assertIn('echo "./post_restore.sh" >> $BACKUPLIST', source) + self.assertIn('[ -e "./extra" ]', source) + + + def test_resize_handlers_preserve_sigwinch(self): + + for relativePath in ( + "scripts/buildstack_menu.py", + "scripts/docker_commands.py", + "scripts/misc_commands.py", + "scripts/backup_restore.py", + "scripts/native_installs.py", + ): + source = (ROOT / relativePath).read_text() + self.assertIn("originalSignalHandler = signal.getsignal(signal.SIGWINCH)", source) + self.assertNotIn("originalSignalHandler = signal.getsignal(signal.SIGINT)", source) + for pythonFile in (ROOT / ".templates").glob("*/*.py"): + self.assertNotIn("getsignal(signal.SIGINT)", pythonFile.read_text(), str(pythonFile)) + def test_backup_and_restore_propagate_archive_failures(self): + backupSource = (ROOT / "scripts/backup.sh").read_text() + restoreSource = (ROOT / "scripts/restore.sh").read_text() + self.assertIn('if ! sudo tar -czf', backupSource) + self.assertIn('if ! tar -tzf "$RESTOREFILE"', restoreSource) + self.assertIn('if ! sudo tar -zxvf', restoreSource) + + + def test_build_menu_does_not_query_cursor_position(self): + source = (ROOT / "scripts/buildstack_menu.py").read_text() + self.assertNotIn("get_location", source) + + def test_build_menu_explains_options_require_selected_container(self): + source = (ROOT / "scripts/buildstack_menu.py").read_text() + self.assertIn("serviceOptionsMessage", source) + transmissionSource = (ROOT / ".templates/transmission/build.py").read_text() + self.assertIn("except OSError as err:", transmissionSource) + self.assertIn('if key and transientMessage:', source) + self.assertIn("if needsRender != 1:", source) + + def test_build_menu_height_is_automatic(self): + source = (ROOT / "scripts/buildstack_menu.py").read_text() + self.assertNotIn("KEY_TAB", source) + self.assertNotIn("paginationExpanded", source) + self.assertIn("pageSizeForTerminal(term.height", source) + self.assertIn("issuePanelHeight(len(issueRows), fixedRows=8)", source) + + def test_hook_failures_stop_the_build(self): + source = (ROOT / "scripts/buildstack_menu.py").read_text() + self.assertIn("if not runPrebuildHook():", source) + self.assertIn("if not runPostBuildHook():", source) + self.assertIn("if hookResult is False:", source) + self.assertIn("postBuildResult = subprocess.call", source) + self.assertIn("if postBuildResult != 0:", source) + self.assertIn("return subprocess.call(commandToRun) == 0", (ROOT / ".templates/nextcloud/build.py").read_text()) + + def test_service_option_renderers_preserve_cursor_and_signal_state(self): + badCursorMove = "print(term.move(hotzoneLocation[0], hotzoneLocation[1]))" + for pythonFile in (ROOT / ".templates").glob("*/*.py"): + source = pythonFile.read_text() + self.assertNotIn(badCursorMove, source, str(pythonFile)) + lines = source.splitlines() + for index, line in enumerate(lines): + if "mainRender(needsRender," not in line: + continue + following = "\n".join(lines[index + 1:index + 4]) + if "SelectionInProgress = True" in following or "selectionInProgress = True" in following: + self.assertIn("needsRender = 0", following, str(pythonFile)) + + for buildScript in (ROOT / ".templates").glob("*/build.py"): + source = buildScript.read_text() + if "signal." in source: + self.assertIn("import signal", source, str(buildScript)) + if buildScript.parent.name != "example_template" and "def runOptionsMenu(context)" in source: + self.assertIn("finally:", source, str(buildScript)) + self.assertIn("signal.signal(signal.SIGWINCH, originalSignalHandler)", source, str(buildScript)) + + def test_docker_commands_only_report_success_on_zero_exit(self): + source = (ROOT / "scripts/docker_commands.py").read_text() + startSource = source[ + source.index("def startStack"):source.index("def restartStack") + ] + self.assertIn('stackStarted = runCommand("docker-compose up -d --remove-orphans")', source) + self.assertIn("if stackStarted:", source) + self.assertIn("return stackStarted", source) + self.assertNotIn('subprocess.call("docker-compose up -d", shell=True)', startSource) + self.assertIn("if not runCommand(command):", source) + self.assertIn("return allStopped", source) + self.assertIn("return volumesPruned", source) + self.assertIn("return imagesPruned", source) + self.assertNotIn('subprocess.call("docker-compose pull"', source) + self.assertNotIn('subprocess.call("docker system prune --volumes"', source) + + def test_required_environment_warnings_are_part_of_build_checks(self): + source = (ROOT / "scripts/buildstack_menu.py").read_text() + self.assertIn("issues.update(requiredEnvironmentIssues(ownedServices))", source) + self.assertIn("Build warning: required environment variables are missing:", source) + self.assertIn('hookState["environmentOptions"]', source) + self.assertIn("runEnvironmentOptions(", source) + self.assertIn("runIssueViewer(term, renderMode, issueEntries)", source) + self.assertIn("elif key.lower() == 'i':", source) + self.assertLess( + source.index("requiredIssues = requiredEnvironmentIssues"), + source.index("if not runPrebuildHook():"), + ) + def test_password_controls_have_one_owner(self): + for serviceName in ("deconz", "influxdb", "mariadb", "nextcloud"): + source = (ROOT / ".templates" / serviceName / "build.py").read_text() + self.assertNotIn("def setPasswordOptions", source, serviceName) + self.assertNotIn("passwords.py", source, serviceName) + self.assertFalse( + (ROOT / ".templates" / serviceName / "passwords.py").exists() + ) + + optionsSource = (ROOT / "scripts/deps/environment_options.py").read_text() + self.assertIn('"Password options (%s)"', optionsSource) + self.assertIn('"Use default: %s"', optionsSource) + self.assertIn('"Enter a custom password"', optionsSource) + self.assertIn('"Generate a random password and save it"', optionsSource) + self.assertIn("Saved in .env.", optionsSource) + self.assertIn("View password saved in .env", optionsSource) + self.assertNotIn("whiptail", optionsSource.lower()) + self.assertNotIn("subprocess", optionsSource) + self.assertIn("with term.cbreak():", optionsSource) + buildMenuSource = (ROOT / "scripts/buildstack_menu.py").read_text() + self.assertIn("configurableEnvironmentVariables(templateServices)", buildMenuSource) + self.assertIn("onValueSaved=onEnvironmentValueSaved", buildMenuSource) + self.assertNotIn("envPath=envFile", buildMenuSource) + self.assertIn('envPath=".env"', optionsSource) + + def test_dynamic_menu_removal_keeps_selection_in_range(self): + + originalMenu = list(menu_main.mainMenuList) + originalAdded = menu_main.potentialMenu["deletePromptFiles"]["added"] + originalIndex = menu_main.currentMenuItemIndex + try: + menu_main.mainMenuList.append(menu_main.potentialMenu["deletePromptFiles"]["menuItem"]) + menu_main.potentialMenu["deletePromptFiles"]["added"] = True + menu_main.currentMenuItemIndex = len(menu_main.mainMenuList) - 1 + self.assertTrue(menu_main.removeMenuItemByLabel("deletePromptFiles")) + self.assertLess(menu_main.currentMenuItemIndex, len(menu_main.mainMenuList)) + finally: + menu_main.mainMenuList[:] = originalMenu + menu_main.potentialMenu["deletePromptFiles"]["added"] = originalAdded + menu_main.currentMenuItemIndex = originalIndex + def test_saved_service_restore_uses_stable_key_snapshot(self): + source = (ROOT / "scripts/buildstack_menu.py").read_text() + self.assertIn("selectedTemplates = [", source) + self.assertIn("for serviceName in selectedTemplates:", source) + + def test_contributor_guide_uses_modern_hook_api(self): + source = (ROOT / "docs/Developers/BuildStack-Services.md").read_text() + self.assertNotIn("HOOK_API_VERSION", source) + self.assertIn("def runChecks(context):", source) + self.assertNotIn("eval(toRun)", source) + self.assertNotIn("buildHooks = {}", source) + + def test_yaml_merge_without_arguments_shows_usage(self): + result = subprocess.run( + [sys.executable, str(ROOT / "scripts/yaml_merge.py")], + capture_output=True, + text=True, + ) + self.assertEqual(4, result.returncode) + self.assertIn("Usage:", result.stdout) + + + def test_default_ports_generator_is_python_and_parses_every_template(self): + script = ROOT / "scripts/default_ports_md_generator.py" + self.assertTrue(script.exists()) + self.assertFalse((ROOT / "scripts/default_ports_md_generator.sh").exists()) + result = subprocess.run( + [sys.executable, str(script)], + capture_output=True, + text=True, + ) + self.assertEqual(0, result.returncode) + self.assertNotIn("Parsing error", result.stdout) + self.assertEqual(63, len(result.stdout.splitlines())) + + +if __name__ == "__main__": + unittest.main()