#Get Results from TMDB for a specific year
def parseTMDBResults(year):
    response = requests.get("https://api.themoviedb.org/3/discover/movie?api_key="+mykey+"&sort_by=popularity.desc&include_adult=false&include_video=true&page=1&primary_release_year="+str(year))
    json_data = json.loads(response.text)
    if debug:
        print "Parsing page 1 from "+str(year)
    parsePage(json_data, False)
    currentpage=2
    totpage=json_data["total_pages"]
    for x in range(2, totpage):
        response = requests.get("https://api.themoviedb.org/3/discover/movie?api_key="+mykey+"&sort_by=popularity.desc&include_adult=false&include_video=true&page="+str(x)+"&primary_release_year="+str(year))
        json_data = json.loads(response.text)
        print "Parsing page " +str(x) + " from "+str(year)
        parsePage(json_data, False)

def getMovieDetails(date,id, update):
        file = open("output/"+date+"_"+str(id)+".json","w" )
        response = requests.get("https://api.themoviedb.org/3/movie/"+str(id)+"?api_key="+mykey+"&append_to_response=credits%2Ckeywords%2Calternative_titles%2Creviews")
        json_data = json.loads(response.text)
        #check for reviews
        json_data["allreviews"]=""
        if "reviews" in json_data:
            reviewresults=json_data["reviews"]["results"]
        else:
            reviewresults=[]
        if (len(reviewresults)>0):
            reviewtext=""
            #we have reviews
            for results in reviewresults:
                reviews= dict()
				#copy parent data from the movie into the review record
                reviews["parentid"]=json_data["id"]
                reviews["id"]=str(json_data["id"])+results["id"]
                reviews["url"]=results["url"]
                reviews["title"]=json_data["title"]
                reviews["release_date"]=json_data["release_date"]
                reviews["content"]=results['content']
                reviews["author"]=results['author']
                reviewtext = reviewtext+"By: "+results["author"]+"<br>"+results["content"]+"<br>"
                filerev = open("output/"+date+"_"+str(id)+"_"+results["id"]+"_review.json","w" )
                filerev.write(json.dumps(reviews).encode('utf-8'))
                filerev.close()
            #We need to sent data to meaningcloud
            if update and reviewtext:
                #send to meaningcloud
                print 'Calling meaningcloud'
                urlmc = "http://api.meaningcloud.com/sentiment-2.1"
                payload = "key=OURAPIKEY&lang=auto&txt="+reviewtext.encode('utf-8')+"&txtf=markup&doc=undefined&sdg=t&dm=s&egp=y&uw=y&rt=n"
                headers = {'content-type': 'application/x-www-form-urlencoded'}

                response = requests.request("POST", urlmc, data=payload, headers=headers)
                
                sentiment= json.loads(response.text)
                time.sleep(0.5)
                sentimentMapping = { "P+": "Strong Positive", "P": "Positive", "ENU": "Neutral", "N":"Negative", "N+":"Strong Negative","NONE":"No sentiment" }
                sentimentAgreeMapping = {"DISAGREEMENT":"Disagreement", "AGREEMENT":"Agreement"}
                sentimentSubjMapping = {"SUBJECTIVE":"Subjective", "OBJECTIVE":"Objective"}
                sentimentIronyMapping = {"IRONIC":"Ironic", "NONIRONIC":"Non-Ironic"}
                if 'score_tag' in sentiment:
                    json_data["mysentimentvalue"]=sentimentMapping[sentiment['score_tag']]
                    json_data["mysentimentagree"]=sentimentAgreeMapping[sentiment['agreement']]
                    json_data["mysentimentsubj"]=sentimentSubjMapping[sentiment['subjectivity']]
                    json_data["mysentimentirony"]=sentimentIronyMapping[sentiment['irony']]

            json_data["allreviews"]=reviewtext
            
            if debug:
                print "We have reviews, creating seperate file."

        json_data["relatedartist"]=""
        json_data["relatedsongs"]=""
		#Scrape the webpage for songs and artists
        if (json_data["imdb_id"]!=""):
            json_data["relatedartist"],json_data["relatedsongs"]=parseIMDBPage(json_data["imdb_id"])


        file.write(json.dumps(json_data).encode('utf-8'))
        file.close()

def parseIMDBPage(id):
    global meta
    #Slow down the parsing
    time.sleep(0.3)
    try:
        html = opener.open("http://www.imdb.com/title/"+str(id)+"/soundtrack")

        bsObj = BeautifulSoup(html.read(),'html.parser')

        content = str(bsObj)
        
    except:
		if debug:
        	print "Error at "+str(id)
        content = ""
        return "", ""
    #print content
    toparse=""
    tosongs=""
    matches = re.finditer(r'Performed by <.[^>]*>(.[^<]*)',content)
    if matches:
        for match in matches:
            if (toparse==""):
                toparse=match.group(1)
            else:
                if toparse.find(match.group(1))==-1:
                    toparse=toparse+";"+match.group(1)

    #also add songs
    matches = re.finditer(r'class=\"soundTrack .[^>]*>(.[^<]*) <br/>',content)
    if matches:
        for match in matches:
            if (tosongs==""):
                tosongs=match.group(1)
            else:
                if tosongs.find(match.group(1))==-1:
                    tosongs=tosongs+";"+match.group(1)
    

    if debug:
        print "IMDB "+str(id)+" Performed by: "+toparse+", songs: "+tosongs

    return toparse, tosongs

#content contains the actual HTML text
compresseddata = zlib.compress(content.encode('utf8'), zlib.Z_BEST_COMPRESSION) # Compress the file content
encodeddata = base64.b64encode(compresseddata)  # Base64 encode the compressed content

def add_document(movie):
	# Use movie id as unique identifier
	meta=dict()
	body=""
	document_id=""
	html= HTMLParser.HTMLParser()
	#We have a normal movie
	document_id = 'https://www.themoviedb.org/movie/'+str(movie['id'])

    #build up all we need in the preview and in the metadata
    # ...

	#crews
	crews=""
	crewsfull=""
	for crew in movie['credits']['crew']:
		if allpeople.find(crew['name'])==-1:
			allpeople=allpeople+crew['name']+';'
		crews=crews+crew['name']+" as "+crew['job']+";" 
		if (crew['profile_path']):
			crewsfull=crewsfull+"<li class='cast'><img class='castimg' src='https://image.tmdb.org/t/p/w66_and_h66_bestv2"+crew['profile_path']+"'><div class='info'><b>"+crew['name']+"</b><br>"+crew['job']+"<br></div></li>"
		else:
			crewsfull=crewsfull+"<li class='cast'><div class='noimage'></div><div class='info'><b>"+crew['name']+"</b><br>"+crew['job']+"<br></div></li>"
	if crewsfull:
		crewsfull="<ol class='castlist'>"+crewsfull+"</ol>"

	# Build up the quickview/preview (HTML)
	content = "<html><head><meta charset='UTF-16'><meta http-equiv='Content-Type' content='text/html; charset=UTF-16'></head>"
	content += "<title>"+movie['title']+"</title>"
	content += "<body>"
	content += "<style>body {    -ms-overflow-style: -ms-autohiding-scrollbar;    background-color: #f4f4f4;    color: #000;    font-family: 'Source Sans Pro', Arial, sans-serif;    font-size: 1em;    -webkit-font-smoothing: antialiased;    -moz-osx-font-smoothing: grayscale;}"
	content += " .header { width: 100%;  position: relative;  z-index: 1;box-sizing:box}"
	# ...
	content += " </style>"
	# ...
	content += "<div class='header_info'><h3>Overview</h3></div>"
	content += "<div><p class='over'>"+movie["overview"]+"</p></div>"
	content += "<div class='header_info'><h3>Other titles</h3></div>"
	content += "<div><p class='over'>"+movie["original_title"]+"</p></div>"
    # ...
	content += "</body></html>"

	body=""
	#Encode the HTML
	compresseddata = zlib.compress(content.encode('utf8'), zlib.Z_BEST_COMPRESSION) # Compress the file content
	encodeddata = base64.b64encode(compresseddata)  # Base64 encode the compressed content
	#For future use if we want to start pushing permissions
	Permissions= json.loads('{"Perms":[{"PermissionSets":[{"AllowAnonymous": "True", "AllowedPermissions":[{"IdentityType":"Group", "Identity": "*@*"}], "DeniedPermissions":[]}]}]}')
	meta["FileExtension"]=".html"
	meta["Permissions"] = Permissions["Perms"]
	meta["CompressedBinaryData"]= encodeddata
	meta["connectortype"]= "Push"
	meta["mytype"]= "Movie"
	meta["myimage"]= movie["poster_path"]
	meta["language"] = "English"
	# ... 
	meta["mystatus"]= movie["status"]
	meta["title"]= movie["title"]
	meta["documentId"]= document_id
	meta["clickableuri"]= document_id
	meta["date"]= movie['release_date']

	body=json.dumps(meta) 

	return body  

def batchPush(jsoncontent):
    #first get S3 link / fileid
    coveo_fileid_api_url = configMovie.get_fileid_api_url()
    coveo_headers = configMovie.get_headers_with_push_api_key()

    print '\n--------\nBATCH, step 1: Call FileId for S3: POST ' + coveo_fileid_api_url
    print 'Headers: ' + str(coveo_headers)

    r = requests.post(coveo_fileid_api_url, headers=coveo_headers)
    fileidjson=json.loads(r.text)
    print "\nResponse from step 1: "+str(r.status_code)

    #create batch json
    print "Response for S3: fileid=>"+fileidjson['fileId']+", uploaduri=>"+fileidjson['uploadUri']
    coveo_batchdocument_api_url = configMovie.get_batch_document_api_url(fileidjson['fileId'])
    
    print '\n---------\nBATCH, setp 2: Upload to s3\n'

    body = "{ \"AddOrUpdate\": [" + jsoncontent + "]}"

    #send it to S3
    r= requests.put(fileidjson['uploadUri'], headers={'Content-Type': 'application/octet-stream','x-amz-server-side-encryption': 'AES256'},data=body)
    print '\nReturn from Upload call: '+str(r.status_code)

    #push to source
    print '\n---------\nBATCH, setp 3: Push call to Cloud\n'

    r = requests.put(coveo_batchdocument_api_url, headers=coveo_headers)
    print '\nReturn from Push call: '+str(r.status_code)

import subprocess
import boto3
import re


db_client = boto3.resource(
    "dynamodb",
    aws_access_key_id="KEY",
    aws_secret_access_key="ACCESS_KEY",
    region_name="YOURREGION"
)
db_table = db_client.Table('ColorsFromImages')


def get_from_cache(key):
    token_item = db_table.get_item(
        Key={'name': key}
    )
    if ('Item' in token_item):
        return token_item['Item']['value']

    return None


def update_cache(key, value):
    db_table.update_item(
        Key={'name': key},
        AttributeUpdates={'value': {'Value': value, 'Action': 'PUT'}}
    )


def get_colors_with_gm(image_path):
    image_name = image_path.replace('/', '')
    curl_cmd = 'curl {}{} -o {} -s'.format(parameters['images_base_path'], image_name, image_name)
    dl_file = subprocess.Popen(curl_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = dl_file.communicate()

    # Command line command and arguments
    COMPRESS_CMDLINE = "gm convert {} -colors 6 -flatten HISTOGRAM:- | gm identify -format '%c' - | sort -r -n".format(image_name)

    convert = subprocess.Popen(COMPRESS_CMDLINE, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = convert.communicate()

    lines = re.sub('\\s', '', out)
    lines = re.sub('srgb\\(', '', lines)
    lines = re.sub(r'(\d+,\d+,\d+),\d+', '\\1', lines)
    m = re.findall(r'\(\d+,\d+,\d+\)', lines)
    image_colors = ' '.join(m)

    update_cache(image_path, image_colors)

    RM_CMDLINE = 'rm {}'.format(image_name)
    convert = subprocess.Popen(RM_CMDLINE, shell=True)

    return image_colors


try:
    image_name = document.get_meta_data_value("myimage")[0]
    log('image_name: ' + image_name)

    image_colors = None
    image_colors = get_from_cache(image_name)

    if image_colors is None:
        log('Getting from GM')
        image_colors = get_colors_with_gm(image_name)

    log('image_colors: ' + image_colors)

    document.add_meta_data({
        "mycolors": image_colors
    })
except Exception as e:
    log(str(e))