SPRNET-1414

Introducing PowerShell scripts designed to enable interaction with S3 for automating the NIGHLY build uploads

Spring.build

@@ -103,7 +103,7 @@ Commandline Examples:
   global DIAGNOSTICS setting, applied to EVERY instance of <copy> task via verbose="${copy-verbose}" attribute
   so as to be toggled in ONE place and affect ALL copy operations; leave FALSE except for debugging/troubleshooting the build
 -->
-  <property name="copy-verbose" value="true" />
+  <property name="copy-verbose" value="false" />
 
   <!-- global project settings -->
   <property name="project.name" value="Spring.NET"/>
@@ -1033,9 +1033,9 @@ Commandline Examples:
     <!-- build all binaries -->
     <call target="build-all-function"/>
     <call target="package-cvs"/>
-	<!-- DISABLED until new strategy for posting build package to site for download can be worked out
+	
 	<call target="upload-cvs-release"/>
-	-->
+	
   </target>
 
   <target name="build-all-function" description="Build what is in cvs taking into account approrpiate clr versions and root target name">
@@ -1192,19 +1192,17 @@ Commandline Examples:
     <property name="serverpart" value="${spring.ssh.server + ':' + spring.web.path}" />
     <property name="nightly.builds.path" value="downloads/nightly"/>
 
-    <echo message="uploading ${filetoupload} to s2-prod-wwworg-p1.vmware.com"/>
-    <exec workingdir="${package.dir}" program="scp_spring02.bat" verbose="true">
-      <arg value="${filetoupload}" />
-      <arg value="${serverpart}/${nightly.builds.path}/${filetoupload}" />
-    </exec>
-
-    <echo message="extracting ${filetoupload} to downloads/latest"/>
-    <property name="extract.command" value="cd website/downloads/nightly/; ./update-latest.sh"/>
-    <exec workingdir="${package.dir}" program="ssh_spring02.bat" verbose="true">
-      <arg value="${spring.ssh.server}" />
-      <arg value="${extract.command}" />
+    <echo message="uploading ${filetoupload} to AmazonS3..."/>
+    <exec workingdir="${spring.base.dir}" program="powershell.exe" verbose="true">
+      <arg value=".\s3-nightly-upload.ps1 -scrFolderPath ${package.dir} -srcFilename ${filetoupload}" />
+    </exec>
+	
+	<echo message="Deleting outdated files on AmazonS3..."/>
+	<exec workingdir="${spring.basedir}" program="powershell.exe" verbose="true">
+      <arg value=".\s3-nightly-delete-old-files.ps1 -leaveLatest 10" />
     </exec>
 
+	
   </target>
 
   <target name="package-release-files" depends="set-package-configuration">

s3-nightly-delete-old-files.ps1

@@ -0,0 +1,83 @@
+Param($leaveLatest)
+#declare named params for ease of calling via command-line
+#note: this line is ONLY valid if it is the first line of the file
+#PLEASE DO NOT MOVE THE PARAM(...) LINE FROM ITS CURRENT POSITION
+# OR THIS SCRIPT WILL BREAK!!!
+
+$credentialsFile = ".\s3-credentials.ps1"
+
+#bail out if the credentials file cannot be found
+if (!([IO.File]::Exists($credentialsFile)))
+{
+	Write-Output ("***ERROR: cannot continue without the S3 credentials contained in " + $credentialsFile)
+	exit 1
+}
+else
+{
+	. $credentialsFile
+}
+
+#CONST values
+$key = GetKey
+$secret = GetSecret
+$targetFolderPath = "dist.springframework.org/snapshot/SPRNET"
+$newline = [Environment]::NewLine
+
+#verify the args aren't null before we proceed
+if ((!($leaveLatest)))
+{
+	Write-Output "***ERROR: INVALID ARGUMENTS***"
+	Write-Output $newline
+	Write-Output "Correct usage is..."
+	Write-Output "powershell.exe <scriptname> -leaveLatest <number-of-files-to-leave>"
+	Write-Output $newline
+	Write-Output "example:"
+	Write-Output "powershell.exe .\delete.ps1 -leaveLatest 10"
+	Write-Output "This will delete all but the 10 most recent files (by date) on S3"
+
+	#exit with exit code 1 (failure) so calling procs can know WTF happened
+	exit 1
+}
+
+
+#ensure the CloudBerry Labs snap-ins are loaded before we try to call them...
+Add-PSSnapin CloudBerryLab.Explorer.PSSnapIn
+
+
+#get the connection to s3
+Write-Output "Connecting to S3..."
+$s3 = Get-CloudS3Connection -Key $key -Secret $secret
+Write-Output "...connection successful."
+Write-Output $newline
+
+#get a folder object that represents the target path
+$targetFolder = $s3 | Select-CloudFolder -Path $targetFolderPath
+
+#get the connection to s3
+Write-Output ("Deleting old uploads so only the most recent " + $leaveLatest + " files remain...") $newline
+$files = $targetFolder | Get-CloudItem | Sort-Object ModifyDate -Descending | Select-Object -Skip $leaveLatest
+
+if($files)
+{
+	$files  | ForEach-Object	{
+									Write-Output ("Removing file: " +  $_.DisplayName + " ( originally uploaded: " + $_.ModifyDate + " )...")
+									Remove-CloudItem -Filter $_.DisplayName -Folder $targetFolder
+									Write-Output "...file deletion successful."
+								}
+}
+else
+{
+	Write-Output (">>>There are less than " + $leaveLatest + " files on S3 so no files need to be removed from S3 at this time.")
+}
+
+
+
+Write-Output $newline
+
+
+
+#if we get this far, we should return ERRORCODE 0 so calling procs can know we succeeded
+#technically this is redundant, but best to be explicit :)
+Write-Output $newline
+Write-Output "Exiting Powershell script normally."
+exit 0

s3-nightly-upload.ps1

@@ -0,0 +1,89 @@
+Param($srcFolderPath, $srcFilename)
+#declare named params for ease of calling via command-line
+#note: this line is ONLY valid if it is the first line of the file
+#PLEASE DO NOT MOVE THE PARAM(...) LINE FROM ITS CURRENT POSITION
+# OR THIS SCRIPT WILL BREAK!!!
+
+$credentialsFile = ".\s3-credentials.ps1"
+
+#bail out if the credentials file cannot be found
+if (!([IO.File]::Exists($credentialsFile)))
+{
+	Write-Output ("***ERROR: cannot continue without the S3 credentials contained in " + $credentialsFile)
+	exit 1
+}
+else
+{
+	. $credentialsFile
+}
+
+#CONST values
+$key = GetKey
+$secret = GetSecret
+$uploadFolderPath = "dist.springframework.org/snapshot/SPRNET"
+$newline = [Environment]::NewLine
+
+
+#verify the args aren't null before we proceed
+if ((!($srcFolderPath))-or (!($srcFilename)))
+{
+	Write-Output "***ERROR: INVALID ARGUMENTS***"
+	Write-Output $newline $newline
+	Write-Output "Correct usage is..."
+	Write-Output "powershell.exe <scriptname> -srcFolderPath <folderpath> -srcFilename <filename>"
+	Write-Output $newline
+	Write-Output "example:"
+	Write-Output "powershell.exe .\s3upload.ps1 -srcFolderPath c:\uploads -srcFilename FileToUpload.txt"
+	
+	#exit with exit code 1 (failure) so calling procs can know WTF happened
+	exit 1
+}
+
+#bail out if the source folder cannot be found
+if (!([IO.Directory]::Exists($srcFolderPath)))
+{
+	Write-Output "***ERROR***"
+	Write-Output ("path '" + $srcFolderPath + "' doesn't exist or in inaccessible.")
+	exit 1
+}
+
+
+#bail out if the source file cannot be found
+
+$fullPathToSourceFile = [IO.Path]::Combine($srcFolderPath, $srcFilename)
+
+if (!([IO.File]::Exists($fullPathToSourceFile)))
+{
+	Write-Output "***ERROR***"
+	
+	Write-Output ("File '" + $fullPathToSourceFile + "' doesn't exist or in inaccessible.")
+	exit 1
+}
+
+#ensure the CloudBerry Labs snap-ins are loaded before we try to call them...
+Add-PSSnapin CloudBerryLab.Explorer.PSSnapIn
+
+
+#get the connection to s3
+Write-Output "Connecting to S3..."
+$s3 = Get-CloudS3Connection -Key $key -Secret $secret
+Write-Output "...connection successful."
+Write-Output $newline
+
+#get a folder object that represents the destination
+$destination = $s3 | Select-CloudFolder -Path $uploadFolderPath
+
+#get a folder object that represents the local source folder
+$source = Get-CloudFilesystemConnection | Select-CloudFolder $srcFolderPath
+
+#perform the actual copy to s3
+Write-Output ("Uploading file '" + $fullPathToSourceFile + "' to S3 path: '" + $uploadFolderPath +"'...")
+$source | Copy-CloudItem $destination $srcFilename
+Write-Output "...upload successful."
+Write-Output $newline
+
+#if we get this far, we should return ERRORCODE 0 so calling procs can know we succeeded
+#technically this is redundant, but best to be explicit :)
+Write-Output $newline
+Write-Output "Exiting Powershell script normally."
+exit 0