Configurable file name (Azure#1323)

* configurable file name * lint & error fix * test fix * fix merge lint errors * e2e and loader test * test linting errors * test linting errors * e2e file name test * test lint checks * testing fixes * moved tests to new file * fixed last test * file name, conciseness * additional tests * slight test changes * updated tests * fixing tests * refactored tests * lint * moved invalid stein tests to broken tests file * fixed test check * sep broken stein tests * renamed, removed prints * edited logs to now also show script file name * formatting * increase timeout * env vars method * printing env variables for failing tests * cls stop env * class methods * setting file name explicitly * reset script file name after test ends * lint + misc fixes * refactor tests * added extra tests * debugging logs * global var * added method to testutils * validate file name + tests * fixed tests * copyright --------- Co-authored-by: Varad Meru <vrdmr@users.noreply.github.com> Co-authored-by: gavin-aguiar <80794152+gavin-aguiar@users.noreply.github.com>

Author: 3 people

azure_functions_worker/constants.py

@@ -41,11 +41,13 @@
 PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39 = True
 PYTHON_EXTENSIONS_RELOAD_FUNCTIONS = "PYTHON_EXTENSIONS_RELOAD_FUNCTIONS"
 
+# new programming model default script file name
+PYTHON_SCRIPT_FILE_NAME = "PYTHON_SCRIPT_FILE_NAME"
+PYTHON_SCRIPT_FILE_NAME_DEFAULT = "function_app.py"
+
 # External Site URLs
 MODULE_NOT_FOUND_TS_URL = "https://aka.ms/functions-modulenotfound"
 
-# new programming model script file name
-SCRIPT_FILE_NAME = "function_app.py"
 PYTHON_LANGUAGE_RUNTIME = "python"
 
 # Settings for V2 programming model

azure_functions_worker/dispatcher.py

@@ -28,14 +28,16 @@
  PYTHON_THREADPOOL_THREAD_COUNT_MAX_37,
  PYTHON_THREADPOOL_THREAD_COUNT_MIN,
  PYTHON_ENABLE_DEBUG_LOGGING,
- SCRIPT_FILE_NAME,
+ PYTHON_SCRIPT_FILE_NAME,
+ PYTHON_SCRIPT_FILE_NAME_DEFAULT,
  PYTHON_LANGUAGE_RUNTIME, CUSTOMER_PACKAGES_PATH)
 from .extension import ExtensionManager
 from .logging import disable_console_logging, enable_console_logging
 from .logging import (logger, error_logger, is_system_log_category,
  CONSOLE_LOG_PREFIX, format_exception)
 from .utils.app_setting_manager import get_python_appsetting_state
-from .utils.common import get_app_setting, is_envvar_true
+from .utils.common import (get_app_setting, is_envvar_true,
+ validate_script_file_name)
 from .utils.dependency import DependencyManager
 from .utils.tracing import marshall_exception_trace
 from .utils.wrappers import disable_feature_by
@@ -327,24 +329,29 @@ async def _handle__worker_status_request(self, request):
  async def _handle__functions_metadata_request(self, request):
  metadata_request = request.functions_metadata_request
  directory = metadata_request.function_app_directory
- function_path = os.path.join(directory, SCRIPT_FILE_NAME)
+ script_file_name = get_app_setting(
+ setting=PYTHON_SCRIPT_FILE_NAME,
+ default_value=f'{PYTHON_SCRIPT_FILE_NAME_DEFAULT}')
+ function_path = os.path.join(directory, script_file_name)
 
  logger.info(
- 'Received WorkerMetadataRequest, request ID %s, directory: %s',
- self.request_id, directory)
-
- if not os.path.exists(function_path):
- # Fallback to legacy model
- logger.info("%s does not exist. "
- "Switching to host indexing.", SCRIPT_FILE_NAME)
- return protos.StreamingMessage(
- request_id=request.request_id,
- function_metadata_response=protos.FunctionMetadataResponse(
- use_default_metadata_indexing=True,
- result=protos.StatusResult(
- status=protos.StatusResult.Success)))
+ 'Received WorkerMetadataRequest, request ID %s, function_path: %s',
+ self.request_id, function_path)
 
  try:
+ validate_script_file_name(script_file_name)
+
+ if not os.path.exists(function_path):
+ # Fallback to legacy model
+ logger.info("%s does not exist. "
+ "Switching to host indexing.", script_file_name)
+ return protos.StreamingMessage(
+ request_id=request.request_id,
+ function_metadata_response=protos.FunctionMetadataResponse(
+ use_default_metadata_indexing=True,
+ result=protos.StatusResult(
+ status=protos.StatusResult.Success)))
+
  fx_metadata_results = self.index_functions(function_path)
 
  return protos.StreamingMessage(
@@ -367,8 +374,6 @@ async def _handle__function_load_request(self, request):
  function_id = func_request.function_id
  function_metadata = func_request.metadata
  function_name = function_metadata.name
- function_path = os.path.join(function_metadata.directory,
- SCRIPT_FILE_NAME)
 
  logger.info(
  'Received WorkerLoadRequest, request ID %s, function_id: %s,'
@@ -377,6 +382,14 @@ async def _handle__function_load_request(self, request):
  programming_model = "V1"
  try:
  if not self._functions.get_function(function_id):
+ script_file_name = get_app_setting(
+ setting=PYTHON_SCRIPT_FILE_NAME,
+ default_value=f'{PYTHON_SCRIPT_FILE_NAME_DEFAULT}')
+ validate_script_file_name(script_file_name)
+ function_path = os.path.join(
+ function_metadata.directory,
+ script_file_name)
+
  if function_metadata.properties.get("worker_indexed", False) \
  or os.path.exists(function_path):
  # This is for the second worker and above where the worker

azure_functions_worker/loader.py

@@ -17,8 +17,10 @@
 
 from . import protos, functions
 from .bindings.retrycontext import RetryPolicy
-from .constants import MODULE_NOT_FOUND_TS_URL, SCRIPT_FILE_NAME, \
- PYTHON_LANGUAGE_RUNTIME, RETRY_POLICY, CUSTOMER_PACKAGES_PATH
+from .utils.common import get_app_setting
+from .constants import MODULE_NOT_FOUND_TS_URL, PYTHON_SCRIPT_FILE_NAME, \
+ PYTHON_SCRIPT_FILE_NAME_DEFAULT, PYTHON_LANGUAGE_RUNTIME, \
+ CUSTOMER_PACKAGES_PATH, RETRY_POLICY
 from .logging import logger
 from .utils.wrappers import attach_message_to_exception
 
@@ -225,7 +227,10 @@ def index_function_app(function_path: str):
  f"level function app instances are defined.")
 
  if not app:
+ script_file_name = get_app_setting(
+ setting=PYTHON_SCRIPT_FILE_NAME,
+ default_value=f'{PYTHON_SCRIPT_FILE_NAME_DEFAULT}')
  raise ValueError("Could not find top level function app instances in "
- f"{SCRIPT_FILE_NAME}.")
+ f"{script_file_name}.")
 
  return app.get_functions()

azure_functions_worker/utils/app_setting_manager.py

@@ -10,7 +10,8 @@
  PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT,
  PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39,
  PYTHON_ENABLE_DEBUG_LOGGING,
- FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED)
+ FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED,
+ PYTHON_SCRIPT_FILE_NAME)
 
 
 def get_python_appsetting_state():
@@ -21,7 +22,8 @@ def get_python_appsetting_state():
  PYTHON_ISOLATE_WORKER_DEPENDENCIES,
  PYTHON_ENABLE_DEBUG_LOGGING,
  PYTHON_ENABLE_WORKER_EXTENSIONS,
- FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED]
+ FUNCTIONS_WORKER_SHARED_MEMORY_DATA_TRANSFER_ENABLED,
+ PYTHON_SCRIPT_FILE_NAME]
 
  app_setting_states = "".join(
  f"{app_setting}: {current_vars[app_setting]} | "

azure_functions_worker/utils/common.py

@@ -3,6 +3,7 @@
 import importlib
 import os
 import sys
+import re
 from types import ModuleType
 from typing import Optional, Callable
 
@@ -136,3 +137,20 @@ def get_sdk_from_sys_path() -> ModuleType:
  sys.path.insert(0, CUSTOMER_PACKAGES_PATH)
 
  return importlib.import_module('azure.functions')
+
+
+class InvalidFileNameError(Exception):
+
+ def __init__(self, file_name: str) -> None:
+ super().__init__(
+ f'Invalid file name: {file_name}')
+
+
+def validate_script_file_name(file_name: str):
+ # First character can be a letter, number, or underscore
+ # Following characters can be a letter, number, underscore, hyphen, or dash
+ # Ending must be .py
+ pattern = re.compile(r'^[a-zA-Z0-9_][a-zA-Z0-9_\-]*\.py$')
+ if not pattern.match(file_name):
+ raise InvalidFileNameError(file_name)
+ return True

tests/endtoend/http_functions/http_functions_stein/file_name/main.py

@@ -0,0 +1,44 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+from datetime import datetime
+import logging
+import time
+
+import azure.functions as func
+
+app = func.FunctionApp(http_auth_level=func.AuthLevel.ANONYMOUS)
+
+
+@app.route(route="default_template")
+def default_template(req: func.HttpRequest) -> func.HttpResponse:
+ logging.info('Python HTTP trigger function processed a request.')
+
+ name = req.params.get('name')
+ if not name:
+ try:
+ req_body = req.get_json()
+ except ValueError:
+ pass
+ else:
+ name = req_body.get('name')
+
+ if name:
+ return func.HttpResponse(
+ f"Hello, {name}. This HTTP triggered function "
+ f"executed successfully.")
+ else:
+ return func.HttpResponse(
+ "This HTTP triggered function executed successfully. "
+ "Pass a name in the query string or in the request body for a"
+ " personalized response.",
+ status_code=200
+ )
+
+
+@app.route(route="http_func")
+def http_func(req: func.HttpRequest) -> func.HttpResponse:
+ time.sleep(1)
+
+ current_time = datetime.now().strftime("%H:%M:%S")
+ return func.HttpResponse(f"{current_time}")

tests/endtoend/test_file_name_functions.py

@@ -0,0 +1,109 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+import os
+import requests
+
+from azure_functions_worker.constants import PYTHON_SCRIPT_FILE_NAME
+from tests.utils import testutils
+
+REQUEST_TIMEOUT_SEC = 10
+
+
+class TestHttpFunctionsFileName(testutils.WebHostTestCase):
+ """Test the native Http Trigger in the local webhost.
+
+ This test class will spawn a webhost from your <project_root>/build/webhost
+ folder and replace the built-in Python with azure_functions_worker from
+ your code base. Since the Http Trigger is a native suport from host, we
+ don't need to setup any external resources.
+
+ Compared to the unittests/test_http_functions.py, this file is more focus
+ on testing the E2E flow scenarios.
+ """
+ @classmethod
+ def get_script_name(cls):
+ return "main.py"
+
+ @classmethod
+ def get_script_dir(cls):
+ return testutils.E2E_TESTS_FOLDER / 'http_functions' / \
+ 'http_functions_stein' / \
+ 'file_name'
+
+ @testutils.retryable_test(3, 5)
+ def test_index_page_should_return_ok(self):
+ """The index page of Azure Functions should return OK in any
+ circumstances
+ """
+ r = self.webhost.request('GET', '', no_prefix=True,
+ timeout=REQUEST_TIMEOUT_SEC)
+ self.assertTrue(r.ok)
+
+ @testutils.retryable_test(3, 5)
+ def test_default_http_template_should_return_ok(self):
+ """Test if the default template of Http trigger in Python Function app
+ will return OK
+ """
+ r = self.webhost.request('GET', 'default_template',
+ timeout=REQUEST_TIMEOUT_SEC)
+ self.assertTrue(r.ok)
+
+ @testutils.retryable_test(3, 5)
+ def test_default_http_template_should_accept_query_param(self):
+ """Test if the azure.functions SDK is able to deserialize query
+ parameter from the default template
+ """
+ r = self.webhost.request('GET', 'default_template',
+ params={'name': 'query'},
+ timeout=REQUEST_TIMEOUT_SEC)
+ self.assertTrue(r.ok)
+ self.assertEqual(
+ r.content,
+ b'Hello, query. This HTTP triggered function executed successfully.'
+ )
+
+ @testutils.retryable_test(3, 5)
+ def test_default_http_template_should_accept_body(self):
+ """Test if the azure.functions SDK is able to deserialize http body
+ and pass it to default template
+ """
+ r = self.webhost.request('POST', 'default_template',
+ data='{ "name": "body" }'.encode('utf-8'),
+ timeout=REQUEST_TIMEOUT_SEC)
+ self.assertTrue(r.ok)
+ self.assertEqual(
+ r.content,
+ b'Hello, body. This HTTP triggered function executed successfully.'
+ )
+
+ @testutils.retryable_test(3, 5)
+ def test_worker_status_endpoint_should_return_ok(self):
+ """Test if the worker status endpoint will trigger
+ _handle__worker_status_request and sends a worker status response back
+ to host
+ """
+ root_url = self.webhost._addr
+ health_check_url = f'{root_url}/admin/host/ping'
+ r = requests.post(health_check_url,
+ params={'checkHealth': '1'},
+ timeout=REQUEST_TIMEOUT_SEC)
+ self.assertTrue(r.ok)
+
+ @testutils.retryable_test(3, 5)
+ def test_worker_status_endpoint_should_return_ok_when_disabled(self):
+ """Test if the worker status endpoint will trigger
+ _handle__worker_status_request and sends a worker status response back
+ to host
+ """
+ os.environ['WEBSITE_PING_METRICS_SCALE_ENABLED'] = '0'
+ root_url = self.webhost._addr
+ health_check_url = f'{root_url}/admin/host/ping'
+ r = requests.post(health_check_url,
+ params={'checkHealth': '1'},
+ timeout=REQUEST_TIMEOUT_SEC)
+ self.assertTrue(r.ok)
+
+ def test_correct_file_name(self):
+ self.assertIsNotNone(os.environ.get(PYTHON_SCRIPT_FILE_NAME))
+ self.assertEqual(os.environ.get(PYTHON_SCRIPT_FILE_NAME),
+ 'main.py')

tests/unittests/broken_functions/invalid_app_stein/function_app.py

@@ -0,0 +1,5 @@
+import azure.functions as func
+
+
+def main(req: func.HttpRequest):
+ pass

tests/unittests/file_name_functions/default_file_name/function_app.py

@@ -0,0 +1,11 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import azure.functions as func
+
+app = func.FunctionApp()
+
+
+@app.route(route="return_str")
+def return_str(req: func.HttpRequest) -> str:
+ return 'Hello World!'

tests/unittests/file_name_functions/invalid_file_name/main

@@ -0,0 +1,11 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import azure.functions as func
+
+app = func.FunctionApp()
+
+
+@app.route(route="return_str")
+def return_str(req: func.HttpRequest) -> str:
+ return 'Hello World!'