Skip to content

Commit

Permalink
Merge pull request #8 from cscaglioned42/master
Browse files Browse the repository at this point in the history 
D42-29213 - GitHub Freshservice - API V1 is being deprecated
Updated relationship v1 API to v2.
  • Loading branch information
@cscaglioned42
cscaglioned42 committed Jun 1, 2023
2 parents b229f91 + 2f7d8a1 commit a0c9a7e
Show file tree
Hide file tree
Showing 3 changed files with 171 additions and 54 deletions.
173 changes: 147 additions & 26 deletions d42_sd_sync.py
View file
Expand Up @@ -11,6 +11,7 @@
import xml.etree.ElementTree as eTree
from xmljson import badgerfish as bf
import time
import math

logger = logging.getLogger('log')
logger.setLevel(logging.INFO)
Expand All @@ -19,6 +20,15 @@
logger.addHandler(ch)
CUR_DIR = os.path.dirname(os.path.abspath(__file__))

RELATIONSHIP_BATCH_SIZE = 20
# With v1 of the API, we were able to create about 4 relationships per second.
# So we will assume that we will be able to create them at the same rate with
# the asynchronous background jobs.
RELATIONSHIPS_CREATED_PER_SECOND = 4
# The number of seconds to wait before we check the status of create relationships jobs.
RELATIONSHIPS_JOB_WAIT_SECONDS = int(math.ceil(RELATIONSHIP_BATCH_SIZE / float(RELATIONSHIPS_CREATED_PER_SECOND)))
ASSET_TYPE_BUSINESS_SERVICE = "Business Service"

parser = argparse.ArgumentParser(description="freshservice")

parser.add_argument('-d', '--debug', action='store_true', help='Enable debug output')
Expand Down Expand Up @@ -225,11 +235,21 @@ def update_objects_from_server(sources, _target, mapping):
data[map_info["@target"]] = value

if existing_object is None:
logger.info("adding device %s" % source["name"])
logger.info("adding asset %s" % source["name"])
new_asset_id = freshservice.insert_asset(data)
logger.info("added new asset %d" % new_asset_id)
else:
logger.info("updating device %s" % source["name"])
logger.info("updating asset %s" % source["name"])
# This is a workaround for an issue with the Freshservice API where if a business service
# asset has the Managed By field filled in and we don't send an agent_id to update this
# field (we don't map any D42 data to this field and shouldn't need to because
# the API will only update the fields that we send), it will result in a validation
# error with the message:
# Assigned agent isn't a member of the group.
# So, if the business service asset has an agent_id already populated, we will send that
# same value over and that will avoid this error.
if _target["@asset-type"] == ASSET_TYPE_BUSINESS_SERVICE and "agent_id" in existing_object and existing_object["agent_id"]:
data["agent_id"] = existing_object["agent_id"]
updated_asset_id = freshservice.update_asset(data, existing_object["display_id"])
logger.info("updated new asset %d" % updated_asset_id)

Expand Down Expand Up @@ -384,16 +404,20 @@ def create_relationships_from_affinity_group(sources, _target, mapping):
logger.info("finished getting all existing devices in FS.")

logger.info("Getting relationship type in FS.")
relationship_type = freshservice.get_relationship_type_by_content(mapping["@forward-relationship"],
mapping["@backward-relationship"])
relationship_type = freshservice.get_relationship_type_by_content(mapping["@downstream-relationship"],
mapping["@upstream-relationship"])
logger.info("finished getting relationship type in FS.")
if relationship_type is None:
log = "There is no relationship type in FS. (%s - %s)" % (
mapping["@forward-relationship"], mapping["@backward-relationship"])
mapping["@downstream-relationship"], mapping["@upstream-relationship"])
logger.info(log)
return

for source in sources:
relationships_to_create = list()
source_count = len(sources)
submitted_jobs = list()

for idx, source in enumerate(sources):
try:
logger.info("Processing %s - %s." % (source[mapping["@key"]], source[mapping["@target-key"]]))
primary_asset = find_object_by_name(existing_objects, source[mapping["@key"]])
Expand All @@ -413,25 +437,122 @@ def create_relationships_from_affinity_group(sources, _target, mapping):
exist = False
for relationship in relationships:
if relationship["relationship_type_id"] == relationship_type["id"]:
if relationship["config_item"]["display_id"] == secondary_asset["display_id"]:
if relationship["secondary_id"] == secondary_asset["display_id"]:
exist = True
break
if exist:
logger.info("There is already relationship in FS.")
continue

data = dict()
data["type"] = "config_items"
data["type_id"] = [secondary_asset["display_id"]]
data["relationship_type_id"] = relationship_type["id"]
data["relationship_type"] = "forward_relationship"
logger.info("adding relationship %s" % source[mapping["@key"]])
new_relationship_id = freshservice.insert_relationship(primary_asset["display_id"], data)
logger.info("added new relationship %d" % new_relationship_id)
relationships_to_create.append({
"relationship_type_id": relationship_type["id"],
"primary_id": primary_asset["display_id"],
"primary_type": "asset",
"secondary_id": secondary_asset["display_id"],
"secondary_type": "asset"
})

# Create a new job if we reached our batch size or we are on the last item (which
# means this is the last batch we will be submitting).
if len(relationships_to_create) >= RELATIONSHIP_BATCH_SIZE or idx == source_count - 1:
submitted_jobs.append(submit_relationship_create_job(relationships_to_create))

# Clear the list for the next batch of relationships we are going to send.
del relationships_to_create[:]
except Exception as e:
log = "Error (%s) creating relationship %s" % (str(e), source[mapping["@key"]])
logger.exception(log)

# We may not have submitted the last batch of relationships to create if the last item in
# sources did not result in a relationship needing to be created (e.g. one of the assets
# in the relationship did not exist in Freshservice, the relationship already existed in
# Freshservice, etc.). So if we have any relationships that we need to create that have
# not been submitted, submit them now.
if relationships_to_create:
submitted_jobs.append(submit_relationship_create_job(relationships_to_create))

del relationships_to_create[:]

if submitted_jobs:
jobs_to_check = list(submitted_jobs)
next_jobs_to_check = list()

# We will make attempts to check the status of the jobs and see if they have
# completed. The max time we will wait is the number of jobs we submitted
# times the amount of time it takes to create a full batch of relationships.
# This total wait time will be broken into chunks based on how long it would
# take a single batch of relationships to be created. For example, if we
# submitted 3 jobs and each job had a batch of 20 relationships to create,
# then it should take 5 seconds to create the 20 relationships based on being
# able to create them at a rate of 4 per second. We will wait 5 seconds, then
# check the status of all jobs. If there are any jobs still waiting to complete,
# then we will wait another 5 seconds and check the status of the jobs that were
# previously waiting to complete.
# Added 20% padding to wait a little bit longer for the jobs to complete
# if needed.
for i in range(int(math.ceil(len(submitted_jobs) * 1.2))):
time.sleep(RELATIONSHIPS_JOB_WAIT_SECONDS)

for job_to_check in jobs_to_check:
try:
job = freshservice.get_job(job_to_check["job_id"])
status = job["status"]

if status == "success":
# All relationships were created.
logger.info("Job %s created all %d relationships successfully." % (job_to_check["job_id"], job_to_check["relationships_to_create_count"]))
elif status in ["failed", "partial"]:
# No relationships were created (failed status) or some relationships
# were created and some were not (partial status).
for relationship in job["relationships"]:
if not relationship["success"]:
log = "Job %s failed to create relationship: %s" % (job_to_check["job_id"], relationship)
logger.error(log)
elif status in ["queued", "in progress"]:
# The job has not completed yet.
next_jobs_to_check.append(job_to_check)
log = "Job %s has not completed yet. The job status is %s." % (job_to_check["job_id"], status)
logger.info(log)
else:
raise Exception("Received unknown job status of %s." % status)
except Exception as e:
log = "Error (%s) checking job %s" % (str(e), job_to_check["job_id"])
logger.exception(log)

# Clear the list.
del jobs_to_check[:]

if next_jobs_to_check:
# We still have jobs we need to check.
jobs_to_check.extend(next_jobs_to_check)

# Clear the list so that we can add the next set of jobs that are
# still waiting to complete.
del next_jobs_to_check[:]
else:
# There are no more jobs that we need to check, so we can stop
# checking.
break

if jobs_to_check:
submitted_jobs_count = len(submitted_jobs)
jobs_not_completed_count = len(jobs_to_check)

logger.info("%d of %d relationship create jobs did not complete." % (jobs_not_completed_count, submitted_jobs_count))


def submit_relationship_create_job(relationships_to_create):
logger.info("adding relationship create job")
# Creating relationships using the v2 API is now an asynchronous operation and is
# performed using background jobs. We will get back the job ID which can then be
# used to query the status of the job.
job_id = freshservice.insert_relationships({"relationships": relationships_to_create})
logger.info("added new relationship create job %s" % job_id)

return {
"job_id": job_id,
"relationships_to_create_count": len(relationships_to_create)
}

def delete_relationships_from_affinity_group(sources, _target, mapping):
global freshservice
Expand All @@ -441,12 +562,12 @@ def delete_relationships_from_affinity_group(sources, _target, mapping):
logger.info("finished getting all existing devices in FS.")

logger.info("Getting relationship type in FS.")
relationship_type = freshservice.get_relationship_type_by_content(mapping["@forward-relationship"],
mapping["@backward-relationship"])
relationship_type = freshservice.get_relationship_type_by_content(mapping["@downstream-relationship"],
mapping["@upstream-relationship"])
logger.info("finished getting relationship type in FS.")
if relationship_type is None:
log = "There is no relationship type in FS. (%s - %s)" % (
mapping["@forward-relationship"], mapping["@backward-relationship"])
mapping["@downstream-relationship"], mapping["@upstream-relationship"])
logger.info(log)
return

Expand All @@ -468,14 +589,14 @@ def delete_relationships_from_affinity_group(sources, _target, mapping):
remove_relationship = None
for relationship in relationships:
if relationship["relationship_type_id"] == relationship_type["id"]:
if relationship["config_item"]["display_id"] == secondary_asset["display_id"]:
if relationship["secondary_id"] == secondary_asset["display_id"]:
remove_relationship = relationship
break
if remove_relationship is None:
logger.info("There is no relationship in FS.")
continue

freshservice.detach_relationship(primary_asset["display_id"], remove_relationship["id"])
freshservice.detach_relationship(remove_relationship["id"])
logger.info("detached relationship %d" % remove_relationship["id"])
except Exception as e:
log = "Error (%s) deleting relationship %s" % (str(e), source[mapping["@key"]])
Expand All @@ -494,12 +615,12 @@ def delete_relationships_from_business_app(sources, _target, mapping):
logger.info("finished getting all existing devices in FS.")

logger.info("Getting relationship type in FS.")
relationship_type = freshservice.get_relationship_type_by_content(mapping["@forward-relationship"],
mapping["@backward-relationship"])
relationship_type = freshservice.get_relationship_type_by_content(mapping["@downstream-relationship"],
mapping["@upstream-relationship"])
logger.info("finished getting relationship type in FS.")
if relationship_type is None:
log = "There is no relationship type in FS. (%s - %s)" % (
mapping["@forward-relationship"], mapping["@backward-relationship"])
mapping["@downstream-relationship"], mapping["@upstream-relationship"])
logger.info(log)
return

Expand All @@ -509,9 +630,9 @@ def delete_relationships_from_business_app(sources, _target, mapping):
relationships = freshservice.get_relationships_by_id(existing_object["display_id"])
for relationship in relationships:
if relationship["relationship_type_id"] == relationship_type["id"] and \
relationship["relationship_type"] == "forward_relationship":
relationship["primary_id"] == existing_object["display_id"]:
remove_relationship = relationship
target_display_id = relationship["config_item"]["display_id"]
target_display_id = relationship["secondary_id"]
for source in sources:
if source[mapping["@key"]] == existing_object["name"]:
secondary_asset = find_object_by_name(existing_objects, source[mapping["@target-key"]])
Expand All @@ -522,7 +643,7 @@ def delete_relationships_from_business_app(sources, _target, mapping):
if remove_relationship is None:
continue

freshservice.detach_relationship(existing_object["display_id"], remove_relationship["id"])
freshservice.detach_relationship(remove_relationship["id"])
logger.info("detached relationship %d" % remove_relationship["id"])
except Exception as e:
log = "Error (%s) deleting relationship %s" % (str(e), existing_object[mapping["@key"]])
Expand Down
44 changes: 20 additions & 24 deletions freshservice.py
View file
Expand Up @@ -263,37 +263,28 @@ def request(self, source_url, method, model):
return models
return []

def get_relationship_type_by_content(self, forward, backward):
path = "/cmdb/relationship_types/list.json"
relationships = None
if relationships is None:
relationships = self._get(path)
def get_relationship_type_by_content(self, downstream, upstream):
path = "/api/v2/relationship_types"
relationship_types = self.request(path, "GET", "relationship_types")

for relationship in relationships:
if relationship["forward_relationship"] == forward and relationship["backward_relationship"] == backward:
return relationship
for relationship_type in relationship_types:
if relationship_type["downstream_relation"] == downstream and relationship_type["upstream_relation"] == upstream:
return relationship_type

return None

def get_relationships_by_id(self, asset_id):
path = "/cmdb/items/%d/relationships.json" % asset_id
result = self._get(path)
if "relationships" in result:
return result["relationships"]
return []

def insert_relationship(self, asset_id, data):
path = "/cmdb/items/%d/associate.json" % asset_id
relationships = self._post(path, data)
if len(relationships) > 0:
return relationships[0]["id"]

return -1
path = "/api/v2/assets/%d/relationships" % asset_id
return self.request(path, "GET", "relationships")

def detach_relationship(self, asset_id, relationship_id):
path = "/cmdb/items/%d/detach_relationship.json" % asset_id
def insert_relationships(self, data):
path = "/api/v2/relationships/bulk-create"
job = self._post(path, data)
return job["job_id"]

return self._delete(path, {"relationship_id": relationship_id})
def detach_relationship(self, relationship_id):
path = "/api/v2/relationships?ids=%d" % relationship_id
return self._delete(path)

def get_installations_by_id(self, display_id):
path = "/api/v2/applications/%d/installations" % display_id
Expand All @@ -320,3 +311,8 @@ def insert_installation(self, display_id, data):
return installation['installation']["id"]

return -1

def get_job(self, job_id):
path = "/api/v2/jobs/%s" % job_id
return self._get(path)

0 comments on commit a0c9a7e

Please sign in to comment.