diff --git a/Algorithmia/algorithm.py b/Algorithmia/algorithm.py index b4f75a1..40be378 100644 --- a/Algorithmia/algorithm.py +++ b/Algorithmia/algorithm.py @@ -38,11 +38,57 @@ def set_options(self, timeout=300, stdout=False, output=OutputType.default, **qu self.query_parameters.update(query_parameters) return self + def get_algorithm_id(self): + url = '/v1/algorithms/' + self.username + '/' + self.algoname + print(url) + api_response = self.client.getJsonHelper(url) + if 'id' in api_response: + return api_response['id'] + else: + raise Exception("field 'id' not found in response: ", api_response) + + + def get_secrets(self): + algorithm_id = self.get_algorithm_id() + url = "/v1/algorithms/" + algorithm_id + "/secrets" + api_response = self.client.getJsonHelper(url) + return api_response + + + def set_secret(self, short_name, secret_key, secret_value, description=None): + algorithm_id = self.get_algorithm_id() + url = "/v1/algorithms/" + algorithm_id + "/secrets" + secret_providers = self.client.get_secret_providers() + provider_id = secret_providers[0]['id'] + + create_parameters = { + "owner_type": "algorithm", + "owner_id": algorithm_id, + "short_name": short_name, + "provider_id": provider_id, + "secret_key": secret_key, + "secret_value": secret_value, + } + if description: + create_parameters['description'] = description + else: + create_parameters['description'] = " " + + print(create_parameters) + api_response = self.client.postJsonHelper(url, create_parameters, parse_response_as_json=True) + return api_response + + # Create a new algorithm - def create(self, details={}, settings={}, version_info={}, source={}, scmsCredentials={}): + def create(self, details, settings, version_info=None, source=None, scmsCredentials=None): url = "/v1/algorithms/" + self.username - create_parameters = {"name": self.algoname, "details": details, "settings": settings, - "version_info": version_info, "source": source, "scmsCredentials": scmsCredentials} + create_parameters = {"name": self.algoname, "details": details, "settings": settings} + if version_info: + create_parameters['version_info'] = version_info + if source: + create_parameters['source'] = source + if scmsCredentials: + create_parameters['scmsCredentials'] = scmsCredentials api_response = self.client.postJsonHelper(url, create_parameters, parse_response_as_json=True) return api_response @@ -60,7 +106,7 @@ def publish(self, details={}, settings={}, version_info={}, source={}, scmsCrede url = "/v1/algorithms/" + self.username + "/" + self.algoname + "/versions" publish_parameters = {"details": details, "settings": settings, "version_info": version_info, "source": source, "scmsCredentials": scmsCredentials} - api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True) + api_response = self.client.postJsonHelper(url, publish_parameters, parse_response_as_json=True, retry=True) return api_response def get_builds(self, limit=56, marker=None): @@ -134,7 +180,7 @@ def versions(self, limit=None, marker=None, published=None, callable=None): def compile(self): # Compile algorithm url = '/v1/algorithms/' + self.username + '/' + self.algoname + '/compile' - response = self.client.postJsonHelper(url, {}, parse_response_as_json=True) + response = self.client.postJsonHelper(url, {}, parse_response_as_json=True, retry=True) return response # Pipe an input into this algorithm diff --git a/Algorithmia/client.py b/Algorithmia/client.py index 74d88fa..dc26e1a 100644 --- a/Algorithmia/client.py +++ b/Algorithmia/client.py @@ -16,7 +16,6 @@ from time import time - class Client(object): 'Algorithmia Common Library' @@ -71,6 +70,11 @@ def username(self): username = next(self.dir("").list()).path return username + def scms(self): + url = "/v1/scms" + response = self.getJsonHelper(url) + return response + def file(self, dataUrl, cleanup=False): if dataUrl.startswith('file://'): return LocalDataFile(self, dataUrl) @@ -172,6 +176,11 @@ def get_supported_languages(self): response = self.getHelper(url) return response.json() + def get_secret_providers(self): + url = "/v1/secret-provider" + api_response = self.getJsonHelper(url) + return api_response + def get_organization_errors(self, org_name): """Gets the errors for the organization. @@ -221,7 +230,7 @@ def report_insights(self, insights): return Insights(insights) # Used internally to post json to the api and parse json response - def postJsonHelper(self, url, input_object, parse_response_as_json=True, **query_parameters): + def postJsonHelper(self, url, input_object, parse_response_as_json=True, retry=False, **query_parameters): headers = {} if self.apiKey is not None: headers['Authorization'] = self.apiKey @@ -253,6 +262,8 @@ def postJsonHelper(self, url, input_object, parse_response_as_json=True, **query return response else: return response + elif retry: + return self.postJsonHelper(url, input_object, parse_response_as_json, False, **query_parameters) else: raise raiseAlgoApiError(response) @@ -283,7 +294,6 @@ def getJsonHelper(self, url, **query_parameters): response = response.json() raise raiseAlgoApiError(response) - def getStreamHelper(self, url, **query_parameters): headers = {} if self.apiKey is not None: diff --git a/Test/api/app.py b/Test/api/app.py index 8871d91..db7efd2 100644 --- a/Test/api/app.py +++ b/Test/api/app.py @@ -60,7 +60,7 @@ async def process_hello_world(request: Request, username, algoname, githash): ### Algorithm Routes @regular_app.get('/v1/algorithms/{username}/{algoname}') -async def process_get_algo(request: Request, username, algoname): +async def process_get_algo(username, algoname): if algoname == "echo" and username == 'quality': return {"id": "21df7a38-eab8-4ac8-954c-41a285535e69", "name": "echo", "details": {"summary": "", "label": "echo", "tagline": ""}, @@ -80,7 +80,8 @@ async def process_get_algo(request: Request, username, algoname): "resource_type": "algorithm"} elif algoname == "echo": return JSONResponse(content={"error": {"id": "1cfb98c5-532e-4cbf-9192-fdd45b86969c", "code": 2001, - "message": "Caller is not authorized to perform the operation"}}, status_code=403) + "message": "Caller is not authorized to perform the operation"}}, + status_code=403) else: return JSONResponse(content={"error": "No such algorithm"}, status_code=404) @@ -102,6 +103,34 @@ async def get_scm_status(username, algoname): return {"scm_connection_status": "active"} +@regular_app.get("/v1/scms") +async def get_scms(): + return {'results': [{'default': True, 'enabled': True, 'id': 'internal', 'name': '', 'provider': 'internal'}, + {'default': False, 'enabled': True, 'id': 'github', 'name': 'https://github.com', + 'provider': 'github', 'scm': {'client_id': '0ff25ba21ec67dbed6e2'}, + 'oauth': {'client_id': '0ff25ba21ec67dbed6e2'}, + 'urls': {'web': 'https://github.com', 'api': 'https://api.github.com', + 'ssh': 'ssh://git@github.com'}}, + {'default': False, 'enabled': True, 'id': 'aadebe70-007f-48ff-ba38-49007c6e0377', + 'name': 'https://gitlab.com', 'provider': 'gitlab', + 'scm': {'client_id': 'ca459576279bd99ed480236a267cc969f8322caad292fa5147cc7fdf7b530a7e'}, + 'oauth': {'client_id': 'ca459576279bd99ed480236a267cc969f8322caad292fa5147cc7fdf7b530a7e'}, + 'urls': {'web': 'https://gitlab.com', 'api': 'https://gitlab.com', + 'ssh': 'ssh://git@gitlab.com'}}, + {'default': False, 'enabled': True, 'id': '24ad1496-5a1d-43e2-9d96-42fce8e5484f', + 'name': 'IQIVA Public GitLab', 'provider': 'gitlab', + 'scm': {'client_id': '3341c989f9d28043d2597388aa4f43ce60a74830b981c4b7d79becf641959376'}, + 'oauth': {'client_id': '3341c989f9d28043d2597388aa4f43ce60a74830b981c4b7d79becf641959376'}, + 'urls': {'web': 'https://gitlab.com', 'api': 'https://gitlab.com', + 'ssh': 'ssh://git@gitlab.com'}}, + {'default': False, 'enabled': False, 'id': '83cd96ae-b1f4-4bd9-b9ca-6f7f25c37708', + 'name': 'GitlabTest', 'provider': 'gitlab', + 'scm': {'client_id': '5e257d6e168d579d439b7d38cdfa647e16573ae1dace6d93a30c5c60b4e5dd32'}, + 'oauth': {'client_id': '5e257d6e168d579d439b7d38cdfa647e16573ae1dace6d93a30c5c60b4e5dd32'}, + 'urls': {'web': 'https://gitlab.com', 'api': 'https://gitlab.com', + 'ssh': 'ssh://git@gitlab.com'}}]} + + @regular_app.get("/v1/algorithms/{algo_id}/errors") async def get_algo_errors(algo_id): return JSONResponse(content={"error": {"message": "not found"}}, status_code=404) @@ -201,9 +230,16 @@ async def compile_algorithm(username, algoname): "resource_type": "algorithm" } +fail_cnt = 0 @regular_app.post("/v1/algorithms/{username}/{algoname}/versions") async def publish_algorithm(request: Request, username, algoname): + global fail_cnt + if "failonce" == algoname and fail_cnt == 0: + fail_cnt +=1 + return JSONResponse(content="This is an expected failure mode, try again", status_code=400) + elif "failalways" == algoname: + return JSONResponse(status_code=500) return {"id": "2938ca9f-54c8-48cd-b0d0-0fb7f2255cdc", "name": algoname, "details": {"summary": "Example Summary", "label": "QA", "tagline": "Example Tagline"}, "settings": {"algorithm_callability": "private", "source_visibility": "open", @@ -415,3 +451,99 @@ async def get_environments_by_lang(language): } ] } + + +@regular_app.get("/v1/secret-provider") +async def get_service_providers(): + return [ + { + "id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "name": "algorithmia_internal_secret_provider", + "description": "Internal Secret Provider", + "moduleName": "module", + "factoryClassName": "com.algorithmia.plugin.sqlsecretprovider.InternalSecretProviderFactory", + "interfaceVersion": "1.0", + "isEnabled": True, + "isDefault": True, + "created": "2021-03-11T20:42:23Z", + "modified": "2021-03-11T20:42:23Z" + } + ] + + +@regular_app.get("/v1/algorithms/{algorithm_id}/secrets") +async def get_secrets_for_algorithm(algorithm_id): + return { + "secrets": [ + { + "id": "45e97c47-3ae6-46be-87ee-8ab23746706b", + "short_name": "MLOPS_SERVICE_URL", + "description": "", + "secret_key": "MLOPS_SERVICE_URL", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-22T14:36:01Z", + "modified": "2022-07-22T14:36:01Z" + }, + { + "id": "50dca60e-317f-4582-8854-5b83b4d182d0", + "short_name": "deploy_id", + "description": "", + "secret_key": "DEPLOYMENT_ID", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:31Z", + "modified": "2022-07-21T19:04:31Z" + }, + { + "id": "5a75cdc8-ecc8-4715-8c4b-8038991f1608", + "short_name": "model_path", + "description": "", + "secret_key": "MODEL_PATH", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:31Z", + "modified": "2022-07-21T19:04:31Z" + }, + { + "id": "80e51ed3-f6db-419d-9349-f59f4bbfdcbb", + "short_name": "model_id", + "description": "", + "secret_key": "MODEL_ID", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:30Z", + "modified": "2022-07-21T19:04:30Z" + }, + { + "id": "8773c654-ea2f-4ac5-9ade-55dfc47fec9d", + "short_name": "datarobot_api_token", + "description": "", + "secret_key": "DATAROBOT_MLOPS_API_TOKEN", + "owner_type": "algorithm", + "owner_id": "fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id": "dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created": "2022-07-21T19:04:31Z", + "modified": "2022-07-21T19:04:31Z" + } + ] + } + + +@regular_app.post("/v1/algorithms/{algorithm_id}/secrets") +async def set_algorithm_secret(algorithm_id): + return { + "id":"959af771-7cd8-4981-91c4-70def15bbcdc", + "short_name":"tst", + "description":"", + "secret_key":"test", + "owner_type":"algorithm", + "owner_id":"fa2cd80b-d22a-4548-b16a-45dbad2d3499", + "provider_id":"dee00b6c-05c4-4de7-98d8-e4a3816ed75f", + "created":"2022-07-22T18:28:42Z", + "modified":"2022-07-22T18:28:42Z" +} \ No newline at end of file diff --git a/Test/regular/algo_failure_test.py b/Test/regular/algo_failure_test.py index 0804b4a..0ec4fc2 100644 --- a/Test/regular/algo_failure_test.py +++ b/Test/regular/algo_failure_test.py @@ -28,3 +28,15 @@ def test_throw_500_error_HTTP_response_on_algo_request(self): result = e pass self.assertEqual(str(self.error_message), str(result)) + + def test_retry_on_400_error_publish(self): + result = self.client.algo("util/failonce").publish() + self.assertEqual(result['version_info']['semantic_version'], "0.1.0") + + def test_throw_on_always_500_publish(self): + try: + result = self.client.algo("util/failalways").publish() + except Exception as e: + result = e + pass + self.assertEqual(str(self.error_message), str(result)) diff --git a/Test/regular/algo_test.py b/Test/regular/algo_test.py index b1da4af..ab63c0d 100644 --- a/Test/regular/algo_test.py +++ b/Test/regular/algo_test.py @@ -20,13 +20,15 @@ def setUpClass(cls): def test_call_customCert(self): result = self.client.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) - self.assertEquals('binary', result.metadata.content_type) - self.assertEquals(bytearray('foo', 'utf-8'), result.result) + self.assertEqual('binary', result.metadata.content_type) + self.assertEqual(bytearray('foo', 'utf-8'), result.result) + + def test_normal_call(self): result = self.client.algo('quality/echo').pipe("foo") - self.assertEquals("text", result.metadata.content_type) - self.assertEquals("foo", result.result) + self.assertEqual("text", result.metadata.content_type) + self.assertEqual("foo", result.result) def test_async_call(self): result = self.client.algo('quality/echo').set_options(output=OutputType.void).pipe("foo") @@ -35,20 +37,20 @@ def test_async_call(self): def test_raw_call(self): result = self.client.algo('quality/echo').set_options(output=OutputType.raw).pipe("foo") - self.assertEquals("foo", result) + self.assertEqual("foo", result) def test_dict_call(self): result = self.client.algo('quality/echo').pipe({"foo": "bar"}) - self.assertEquals("json", result.metadata.content_type) - self.assertEquals({"foo": "bar"}, result.result) + self.assertEqual("json", result.metadata.content_type) + self.assertEqual({"foo": "bar"}, result.result) def test_algo_exists(self): result = self.client.algo('quality/echo').exists() - self.assertEquals(True, result) + self.assertEqual(True, result) def test_algo_no_exists(self): result = self.client.algo('quality/not_echo').exists() - self.assertEquals(False, result) + self.assertEqual(False, result) #TODO: add more coverage examples to check kwargs def test_get_versions(self): @@ -56,19 +58,19 @@ def test_get_versions(self): self.assertTrue('results' in result) self.assertTrue('version_info' in result['results'][0]) self.assertTrue('semantic_version' in result['results'][0]['version_info']) - self.assertEquals('0.1.0', result['results'][0]['version_info']['semantic_version']) + self.assertEqual('0.1.0', result['results'][0]['version_info']['semantic_version']) def test_text_unicode(self): telephone = u"\u260E" # Unicode input to pipe() result1 = self.client.algo('quality/echo').pipe(telephone) - self.assertEquals('text', result1.metadata.content_type) - self.assertEquals(telephone, result1.result) + self.assertEqual('text', result1.metadata.content_type) + self.assertEqual(telephone, result1.result) # Unicode return in .result result2 = self.client.algo('quality/echo').pipe(result1.result) - self.assertEquals('text', result2.metadata.content_type) - self.assertEquals(telephone, result2.result) + self.assertEqual('text', result2.metadata.content_type) + self.assertEqual(telephone, result2.result) def test_algo_info(self): result = self.client.algo('quality/echo').info() @@ -175,6 +177,16 @@ def test_algorithm_programmatic_create_process(self): self.assertEqual(response['version_info']['semantic_version'], "0.1.0", "information is incorrect") + + def test_set_secret(self): + short_name = "tst" + secret_key = "test_key" + secret_value = "test_value" + description = "loreum epsum" + response = self.client.algo("quality/echo").set_secret(short_name, secret_key, secret_value, description) + self.assertEqual(response['id'], "959af771-7cd8-4981-91c4-70def15bbcdc", "invalid ID for created secret") + + else: class AlgoTest(unittest.TestCase): def setUp(self): @@ -184,8 +196,8 @@ def test_call_customCert(self): open("./test.pem", 'w') c = Algorithmia.client(ca_cert="./test.pem") result = c.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) - self.assertEquals('binary', result.metadata.content_type) - self.assertEquals(bytearray('foo', 'utf-8'), result.result) + self.assertEqual('binary', result.metadata.content_type) + self.assertEqual(bytearray('foo', 'utf-8'), result.result) try: os.remove("./test.pem") except OSError as e: @@ -193,8 +205,8 @@ def test_call_customCert(self): def test_call_binary(self): result = self.client.algo('quality/echo').pipe(bytearray('foo', 'utf-8')) - self.assertEquals('binary', result.metadata.content_type) - self.assertEquals(bytearray('foo', 'utf-8'), result.result) + self.assertEqual('binary', result.metadata.content_type) + self.assertEqual(bytearray('foo', 'utf-8'), result.result) def test_async_call(self): result = self.client.algo('quality/echo').set_options(output=OutputType.void).pipe("foo") @@ -203,7 +215,7 @@ def test_async_call(self): def test_raw_call(self): result = self.client.algo('quality/echo').set_options(output=OutputType.raw).pipe("foo") - self.assertEquals("foo", result) + self.assertEqual("foo", result) #TODO: add more coverage examples to check kwargs def test_get_versions(self): @@ -211,20 +223,20 @@ def test_get_versions(self): self.assertTrue('results' in result) self.assertTrue('version_info' in result['results'][0]) self.assertTrue('semantic_version' in result['results'][0]['version_info']) - self.assertEquals('0.1.0', result['results'][0]['version_info']['semantic_version']) + self.assertEqual('0.1.0', result['results'][0]['version_info']['semantic_version']) def test_text_unicode(self): telephone = u"\u260E" # Unicode input to pipe() result1 = self.client.algo('quality/echo').pipe(telephone) - self.assertEquals('text', result1.metadata.content_type) - self.assertEquals(telephone, result1.result) + self.assertEqual('text', result1.metadata.content_type) + self.assertEqual(telephone, result1.result) # Unicode return in .result result2 = self.client.algo('quality/echo').pipe(result1.result) - self.assertEquals('text', result2.metadata.content_type) - self.assertEquals(telephone, result2.result) + self.assertEqual('text', result2.metadata.content_type) + self.assertEqual(telephone, result2.result) def test_get_scm_status(self): diff --git a/Test/regular/client_test.py b/Test/regular/client_test.py index c7ed9f2..9cfc39b 100644 --- a/Test/regular/client_test.py +++ b/Test/regular/client_test.py @@ -20,6 +20,7 @@ class ClientDummyTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.client = Algorithmia.client(api_address="http://localhost:8080", api_key="simabcd123") + admin_username = "a_Mrtest" admin_org_name = "a_myOrg" environment_name = "Python 3.9" @@ -28,7 +29,6 @@ def setUp(self): self.admin_username = self.admin_username + str(int(random() * 10000)) self.admin_org_name = self.admin_org_name + str(int(random() * 10000)) - def test_create_user(self): response = self.client.create_user( {"username": self.admin_username, "email": self.admin_username + "@algo.com", "passwordHash": "", @@ -60,6 +60,12 @@ def test_get_environment(self): if u'error' not in response: self.assertTrue(response is not None and u'environments' in response) + def test_get_scms(self): + response = self.client.scms() + results = response['results'] + internal = [result for result in results if result['id'] == 'internal'] + self.assertTrue(len(internal) == 1) + def test_edit_org(self): org_name = "a_myOrg84" @@ -120,7 +126,6 @@ def test_get_algorithm_errors(self): except AlgorithmException as e: self.assertTrue(e.message == "No such algorithm") - def test_no_auth_client(self): key = os.environ.get('ALGORITHMIA_API_KEY', "") @@ -135,6 +140,7 @@ def test_no_auth_client(self): error = e finally: os.environ['ALGORITHMIA_API_KEY'] = key - self.assertEqual(str(error), str(AlgorithmException(message="authorization required", stack_trace=None, error_type=None))) + self.assertEqual(str(error), str(AlgorithmException(message="authorization required", stack_trace=None, + error_type=None))) if __name__ == '__main__': unittest.main() diff --git a/requirements.txt b/requirements.txt index f52fe4f..207a7f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ enum-compat toml argparse algorithmia-api-client==1.5.1 -algorithmia-adk>=1.2,<1.3 +algorithmia-adk>=1.2,<1.4 numpy<2 uvicorn==0.14.0 fastapi==0.65.2 diff --git a/requirements27.txt b/requirements27.txt index 9668467..8a118ea 100644 --- a/requirements27.txt +++ b/requirements27.txt @@ -4,5 +4,5 @@ enum-compat toml argparse algorithmia-api-client==1.5.1 -algorithmia-adk>=1.2,<1.3 +algorithmia-adk>=1.2,<1.4 numpy<2 diff --git a/setup.py b/setup.py index aae416c..0069b73 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ 'toml', 'argparse', 'algorithmia-api-client==1.5.1', - 'algorithmia-adk>=1.2,<1.3' + 'algorithmia-adk>=1.2,<1.4' ], include_package_data=True, classifiers=[