# name=ESP_Import_Russian
# displayinmenu=true
# displaytouser=true
# displayinselector=true
from javax.swing     import JOptionPane
from java.util       import ArrayList
from java.util       import Calendar,TimeZone
from hec.heclib.dss  import HecDss
from hec.heclib.util import HecTime
from hec.hecmath     import TimeSeriesMath
from hec.io          import TimeSeriesContainer
from hec.script      import Constants
from urllib2         import urlopen,HTTPError
from StringIO        import StringIO
from zipfile         import ZipFile
from datetime        import datetime
import os, sys,io 
import	csv
##############CAVI SETUP###################
try :
	from usace.cavi.client import CAVI as cCAVI
	from usace.cavi.script import CAVI as sCAVI
	has_cavi      = True
	cavi_frame    = cCAVI.getBrowserFrame()
	proj_dir      = sCAVI.getWatershed().getProjectDirectory()
	cwms_home     = os.path.join(proj_dir, "..", "..")
	dss_filename  = "CNRFC_ensembles.dss"
	dss_location  = os.path.join(cwms_home, "database")
	script_dir    = os.path.join(proj_dir, "scripts")
	
except ImportError :
	has_cavi      = False
	dss_filename  = None
		


################ Config ################
# The CNRFC Region to fetch, case sensitive (matches name in URL)
Watershed = "russian"

GagesOfInterest = [
	"UKAC1",
	"HOPC1",
	"CDLC1",
	"GEYC1",
	"HEAC1",
	"RIOC1",
	"GUEC1",
	"LAMC1",
	"WSDC1",
		]
# "CXLC1", "YTPC1", "YDMC1", "TLMC1", "JIMC1", "RRHC1", "IRAC1", "RIOC1","GUIC1", "RROC1",
################ Code ##################

def readCSV(fileName):
	with open(fileName) as f:
		data=list(csv.reader(f))
	return data

# Reads an archive ensemble, and returns the data only for the gage desired
def readArchiveCSV(fileName, gage):
	with open(fileName) as f:
		csv_data = csv.reader(f)
		start = -1
		end = -1

		found_start = False
		#header = [csv_data[0][0]]
		# Get the columns that match the data we want
		header = csv_data.next()
		header_row = [header[0]]
		for i in range(1, len(header)):
			if not found_start and header[i] == gage:
				start = i
				found_start = True
			elif found_start and header[i] != gage:
				end = i
				break
			
			if found_start:
				header_row = header_row + [header[i]]

		if start != -1:
			# gage wasn't in file
			return []

		output = io.BytesIO()
		writer = csv.writer(output)

		writer.writerow(header_row)

		# write the specific columns we care about
		for row_in in csv_data:
			row_out = [row_in[0]] + row_in[start:end]
			writer.writerow(row_out)

		data = list(csv.reader(output))
	return data

def downloadFile(url,destinationFileName):
	r = urlopen(url)
	data = r.read()
	with open(destinationFileName,'wb') as f:
		f.write(data)

def downloadZip(url,destinationFileName):
	if os.path.exists(destinationFileName):
		return
	r = urlopen(url)
	zip = ZipFile(StringIO(r.read()))
	zip.extract(os.path.basename(destinationFileName), os.path.dirname(destinationFileName))

def createTemplateTSC(rawDataList):
	#Derives a TimeSeriesContainer object from the raw ESP data list
	#  where all that needs to be done is update the pathname
	#  and values - timestamps should be uniform across each 
	#  ESP trace
  
	#intializing HEC java objects
	tsc = TimeSeriesContainer() #new TSC object
	cal = Calendar.getInstance(TimeZone.getTimeZone(Timezone))
	hecStartTime=  HecTime(cal)
	hecEndTime =  HecTime(cal)

	#copmuting HEC times and interval (minutes) of timestep
	times = []
	for i in range(len(rawDataList)):
		times.append(rawDataList[i][0])
	hecStartTime.set(times[0])
	hecEndTime.set(times[-1])
	
	#The formatting of these times might need to be adjusted at a later point
	
	Dates = []
	for x in range(len(times)):
		IndividualDate = times[x]
		T = HecTime()
		T.set(IndividualDate)
		Dates.append(T.value())

	DiffBetweenInterval = []
	DiffBetweenInterval = [a - Dates[i-1] for i, a in enumerate(Dates)][1:]
	for x in DiffBetweenInterval:
		UniqueList_Minutes = []
		#Check if exist in list or not
		if x not in UniqueList_Minutes:
			UniqueList_Minutes.append(x)
	interval =UniqueList_Minutes[0]
	hecTimes = list(range(Dates[0],Dates[-1],int(interval)))
	hecTimes.append(Dates[-1]) 
	interval_hours = int(interval)/60
	tsc.setTimeZoneID(Timezone)
	tsc.times = hecTimes
	tsc.values = [Constants.UNDEFINED]*len(times) #add null data number here.
	tsc.interval = interval_hours
	tsc.numberValues = len(times)
	tsc.units = "CFS"
	tsc.type = "INST-VAL"
	tsc.parameter =  "FLOW" #Assuming always want this to be flow
	
	return tsc
	
def getESPConfigFromURL(gage):
	if gage[-1] == 'L':
		return "local"
	elif gage[-1] == 'F':
		return("natural")
	else:
		return("regulated")
	#This was orginally set using a couple of different options but this code is being written in a way to only return "natural" values
	
	#This is R code, needs to be modified if desired to be used in Jython.
	#rfcLinks$esp_config[rfcLinks$url==siteURL]

def getESPFlowFromURL(gage):
	if gage[-1] == 'L':
		return "-Loc"
	elif gage[-1] == 'F':
		return("-Nat")
	else:
		return("")

def MultValues(CSV_Data, colName):
	Values = []
	TempVal = []
	for i in range(len(CSV_Data)):
		TempValInt = CSV_Data[i]
		MultCol = float(TempValInt)*1000
		Values.append(round(MultCol,2))	
	return Values

def formDSSPath(colName, gage, interval='1HOUR'):
	#Forms an appropriate DSS file name from the URL and the column name
	#Traces start at 1
	#output (e.g.): //ALF/FLOW/01JUL2019/1HOUR/C:000001
	siteID = ESP_TraceOfInterest.rsplit('.',2)[0].rsplit("_",1)[0] #B part
	trace = 1 + int( colName) #F part
	esp_config = getESPConfigFromURL(gage) #esp type in F part
	esp_flow = getESPFlowFromURL(gage) #esp type in F part
	return("/%s/%s/FLOW%s//%s/C:%06d|CNRFC/" %( Watershed.upper(), siteID, esp_flow.upper(), interval.upper(), trace))

def updateTSCfromESP(tsc, values, colName, gage):
	path = formDSSPath(colName,gage,Interval)
	tsc.values = MultValues(values,colName)
	tsc.fullName = path
	tsc.location = getPathPart(path,"A")
	tsc.watershed = getPathPart(path,"B")
	tsc.version = getPathPart(path,"F")
	return tsc

def AlphaCharacterNum(DSSPart):
	if DSSPart == "A":
		return(1)
	elif DSSPart == "B":
		return(2)
	elif DSSPart == "C":
		return(3)
	elif DSSPart == "D":
		return(4)
	elif DSSPart == "E":
		return(5)
	elif DSSPart == "F":
		return(6)

def getPathPart(path,part): #gets A-F part from DSS path
	partIndex = AlphaCharacterNum(part)
	return(path.split('/')[partIndex])

def GetEnsembleValues(ESP_Matrix,columnToExtract):
	rval=[]
	for i in range(len(ESP_Matrix)):
	        TempESP_Value = ESP_Matrix[i][columnToExtract]	        
	        rval.append(float(TempESP_Value))
	return rval

def GagesToDownload(EspDaysCSV):
	for x in range(len(EspDaysCSV)):
		for i in range(len(GagesOfInterest)):
			if EspDaysCSV[x][:5].upper == GagesOfInterest[i][:5].upper:		
				GagesUsed.append(EspDaysCSV[x])
	return GagesUsed
	
def SaveToDSS(rawDataList,SaveLocation,gage):
	 #From raw data list, saves the list of ESP dataframes to file
	#  If saveLocation is a file, saves there.  saveLocation can also
	#  be a directory, in which case a default file name is written provided
	#  the espDays  argument is assigned	
	if SaveLocation[-3:].upper()=='DSS':
		outFileName =  ("%s" %(SaveLocation))
	else:
		outFileName = ("%s\\rfc_esp_flows.dss" % SaveLocation)
	dssFile = HecDss.open(outFileName,1)

	ESP_Matrix = rawDataList[2:] #This is set for NW RFC's data. There is extra meta data that we do not need to parse through

	TimeStamp_Matrix = []
	for i in range(len(ESP_Matrix)):
		TempTimeStampData = ESP_Matrix[i][0]
		TimeStamp_Matrix.append(TempTimeStampData)
	#print(TimeStamp_Matrix)

	EnsembleDataValues = []
	for i in range(len(ESP_Matrix)):
		TempESP_Value = ESP_Matrix[i][1:]
		EnsembleDataValues.append(TempESP_Value)
	
	templateTsc = createTemplateTSC(ESP_Matrix)
	
	#NumMembers is the variable to represent the number of ensemble members. This currently has the minus 1 to remove the initial column that is labeling the second dimision of the data.
	NumMembers = len(ESP_Matrix[0])-1
	#NumTimeSteps is to denote how many time steps are with the data. 
	NumTimeSteps = len(ESP_Matrix)

	for colIndex in range(NumMembers):
		tsc = updateTSCfromESP(tsc = templateTsc, #update tsc object with data for this ESP trace
		                      values = GetEnsembleValues(ESP_Matrix,colIndex+1),
		                      colName = colIndex,
		                      gage = gage)
		dssFile.put(tsc)
	dssFile.done()

################# Main ##########################
script_directory = os.path.dirname(os.path.abspath(sys.argv[0]))
os.chdir(script_directory)

#This is to set the file location.
#It allows for the user to choose if they want to use the default open DSS File or if they want to use a different file location.
#choices = ( "User Input", "Same as Open DSS File" )
#DssFileLocation = JOptionPane.showInputDialog(None, "How would you like to denote the DSS file path?","DSS File Path", JOptionPane.QUESTION_MESSAGE, None,  choices, choices[0])
DssFileLocation ="User Input" 
if DssFileLocation == "User Input":
	#SaveLocation = JOptionPane.showInputDialog( "Please input a save location (Do not end with '\\')'.\nYou can either put in a DSS file location to specify a name or a file location and a default name will be popualted")
	SaveLocation = "CNRFC_ensembles.dss"
elif DssFileLocation == "Same as Open DSS File":
	None
	#Want this to be able to automatically direct to the open DSS file. 
#GagesOfInterest = JOptionPane.showInputDialog( "Please input the gage identifiers (seperate with a common for multiple locations").rsplit(',')
SaveLocation = os.path.join(dss_location, SaveLocation)
FileNames = []

# This is the timezone the source data is in, _not_ the timezone you want to see it in.
Timezone = "UTC"

# Archive filename template for CNRFC
ArchiveName = "%s_%s_hefs_csv_%s"

print("\n\nBeginning ESP Download\n")

#if not os.path.exists(SaveLocation):
#	os.mkdir(SaveLocation)

for Interval in ['1HOUR']:
	Missing = []

	if Interval == "1HOUR":
		cnrfc_interval = 'hourly'
	else:
		cnrfc_interval = 'daily'

	for gage in GagesOfInterest:
		FileName = "%s_hefs_csv_%s.csv" % (gage, cnrfc_interval)
		CaliforniaNevadaRFC_URL = "https://www.cnrfc.noaa.gov/csv/"

		GagesUsed = []
		GagesToDownload(GagesOfInterest)

		#CheckOnSpelling(GagesUsed)
		#Want to use a function to throw an exception if the infroamtion the user inputted is not within the list.
		#This is not working at this time, so if the user does not type the location in like denoted in the list, an error in the script will occur
		# and is only visible in the console. This error is not the best if the user does not understand how the code is working and parsing the data.
		
		ESP_TraceOfInterest = gage
		webrequest = CaliforniaNevadaRFC_URL + FileName
		
		#Temp Download process
		FileLocation = os.path.join(SaveLocation.rsplit(os.sep,1)[0], 'temp')
		if not os.path.exists(FileLocation):
			os.mkdir(FileLocation)

		FileNames.append(FileName)
		print("\nDownloading %s" % webrequest)
		try:
			downloadFile(webrequest, os.path.join(FileLocation, FileName))
			#Data manipulation 
			data = readCSV(os.path.join(FileLocation, FileName))
		except HTTPError as e:
			sys.stderr.write("Error downloading for '%s'\n" % gage)
			# Try the archive file
			filename = ArchiveName % (datetime.now().strftime('%Y%m%d') + '12', Watershed, cnrfc_interval)
			FileName = filename + '.csv'
			webrequest = CaliforniaNevadaRFC_URL + filename + ".zip"
			print("\nDownloading %s" % webrequest)
			downloadZip(webrequest, os.path.join(FileLocation, FileName))
			
			data = readArchiveCSV(os.path.join(FileLocation, FileName), gage)
		
		#Function to do converstions
		try:
			SaveToDSS(data, SaveLocation, gage)
		except Exception as e:
			Missing.append(gage)

	if len(Missing) > 0:
		sys.stderr.write("\nCould not find the following gages: %s\n" % ", ".join(Missing))

#This has an addition of webrequest as an input to the "SaveToDSS" function call.

print("-"*100)
print("Defined Parameters")
print("DSS File Location: %s" %SaveLocation)
#print("ESP Days: %s" %EspDays)
print("File Location: %s" %FileLocation)
print("File Names: %s" %FileNames)
print("-"*100)

JOptionPane.showMessageDialog(None , "The script has finished collecting the data and placed into the designated dss file.")

# vim: ts=4 sw=4 noexpandtab
