Python实现葵花8号卫星数据自动下载实例

  /pub/himawari/l3/arp/031/

# -*- codeing = utf-8 -*-
# 可以部署在日本的服务器上，下载速度很快
import ftplib
import json
import os
import time
import numpy as np
debuglocaldownload = true
debugdownloaddaily = false
globpersonaltime = [2022, 9, 7]
class numpyencoder(json.jsonencoder):
    def default(self, obj):
        if isinstance(obj, (np.int_, np.intc, np.intp, np.int8,
                            np.int16, np.int32, np.int64, np.uint8,
                            np.uint16, np.uint32, np.uint64)):
            return int(obj)
        elif isinstance(obj, (np.float_, np.float16, np.float32, np.float64)):
            return float(obj)
        elif isinstance(obj, (np.ndarray,)):
            return obj.tolist()
        return json.jsonencoder.default(self, obj)
class himawari:
    ftp = ftplib.ftp()
    def __init__(self):
        self._url = '/pub/himawari/l3/arp/031/'
        self._save_path = './your_save_path'
        if debuglocaldownload:
            self._save_path = './download/'
        self.ftp.connect('ftp.ptree.jaxa.jp', 21)
        self.ftp.login('yourftpaccount', 'yourftppassword')
        self._yearnum, self._monnum, self._daynum = self.dayinit()
        self._nginx_path = ''
        print(self.ftp.welcome)  # 显示登录信息
    def run(self):
        self._nginx_path = ''
        try:
            if debugdownloaddaily:
                self._yearnum, self._monnum, self._daynum = self.getyesterday(self._yearnum, self._monnum, self._daynum)
            else:
                self._yearnum = globpersonaltime[0]
                self._monnum = globpersonaltime[1]
                self._daynum = globpersonaltime[2]
            self._yearstr, self._monstr, self._daystr = self.getdatestr(self._yearnum, self._monnum, self._daynum)
            ftp_filepath = self._url + self._yearstr + self._monstr + "/" + self._daystr + "/"
            # 从目标路径ftp_filepath将文件下载至本地路径dst_filepath
            dst_filepath = self._nginx_path + self._save_path + self._yearstr + "/" + self._monstr + "/" + self._daystr + "/" + "hour" + "/"
            self.deletefile(dst_filepath)  # 先删除未下载完成的临时文件
            print("local:" + dst_filepath)
            print("remote:" + ftp_filepath)
            self.downloadfiletree(dst_filepath, ftp_filepath)
            if debugdownloaddaily:
                self.ftp.quit()
        except exception as err:
            print(err)
    def getyesterday(self, yy, mm, dd):
        dt = (yy, mm, dd, 9, 0, 0, 0, 0, 0)
        dt = time.mktime(dt) - 86400
        yesterdaylist = time.strftime("%y-%m-%d", time.localtime(dt)).split('-')
        return int(yesterdaylist[0]), int(yesterdaylist[1]), int(yesterdaylist[2])
    def dayinit(self, ):
        yesterdaylist = time.strftime("%y-%m-%d", time.localtime(time.time())).split('-')
        return int(yesterdaylist[0]), int(yesterdaylist[1]), int(yesterdaylist[2])
    def getdatestr(self, yy, mm, dd):
        syy = str(yy)
        smm = str(mm)
        sdd = str(dd)
        if mm < 10:
            smm = '0' + smm
        if dd < 10:
            sdd = '0' + sdd
        return syy, smm, sdd
    # 删除目录下扩展名为.temp的文件
    def deletefile(self, filedir):
        if os.path.isdir(filedir):
            targetdir = filedir
            for file in os.listdir(targetdir):
                targetfile = os.path.join(targetdir, file)
                if targetfile.endswith('.temp'):
                    os.remove(targetfile)
    # 下载单个文件，localfile表示本地存储路径和文件名，remotefile是ftp路径和文件名
    def downloadfile(self, localfile, remotefile):
        bufsize = 102400
        file_handler = open(localfile, 'wb')
        print(file_handler)
        # 接收服务器上文件并写入本地文件
        self.ftp.retrbinary('retr ' + remotefile, file_handler.write, bufsize)
        self.ftp.set_debuglevel(0)
        file_handler.close()
        return true
    # 下载整个目录下的文件，localdir表示本地存储路径， emotedir表示ftp路径
    def downloadfiletree(self, localdir, remotedir):
        # 如果本地不存在该路径，则创建
        if not os.path.exists(localdir):
            os.makedirs(localdir)
            # 获取ftp路径下的全部文件名，以列表存储
        self.ftp.cwd(remotedir)
        remotenames = self.ftp.nlst()
        remotenames.reverse()
        # print("remotenames：", remotenames)
        for file in remotenames:
            # 先下载为临时文件local,下载完成后再改名为nc4格式的文件
            # 这是为了防止上一次下载中断后，最后一个下载的文件未下载完整，而再开始下载时，程序会识别为已经下载完成
            local = os.path.join(localdir, file[0:-3] + ".temp")
            files = file[0:-3] + ".nc"
            localnew = os.path.join(localdir, files)
            '''
            下载小时文件，只下载utc时间0时至24时（北京时间0时至24时）的文件
            下载的文件必须是nc格式
            若已经存在，则跳过下载
            '''
            # 小时数据命名格式示例：h08_20200819_0700_1harp030_fldk.02401_02401.nc
            if int(file[13:15]) >= 0 and int(file[13:15]) <= 24:
                if not os.path.exists(localnew):
                    #print("downloading the file of %s" % file)
                    self.downloadfile(local, file)
                    os.rename(local, localnew)
                    print("the download of the file of %s has finished\n" % file)
                    #print("png of the file of %s has finished\n" % png_name)
                elif os.path.exists(localnew):
                    print("the file of %s has already existed!\n" % file)
        self.ftp.cwd("..")
        return
# 主程序
myftp = himawari()
if debugdownloaddaily:
    myftp.run()
else:
    yystart, mmstart, ddstart = input("start(yy mm dd):").split()
    yystart, mmstart, ddstart = int(yystart), int(mmstart), int(ddstart)
    yyend, mmend, ddend = input("end(yy mm dd):").split()
    yyend, mmend, ddend = int(yyend), int(mmend), int(ddend)
    dtstart = (yystart, mmstart, ddstart, 9, 0, 0, 0, 0, 0)
    dtend = (yyend, mmend, ddend, 10, 0, 0, 0, 0, 0)
    timeindex = time.mktime(dtstart)
    timeindexend = time.mktime(dtend)
    while timeindex < timeindexend:
        indexdaylist = time.strftime("%y-%m-%d", time.localtime(timeindex)).split('-')
        globpersonaltime[0] = int(indexdaylist[0])
        globpersonaltime[1] = int(indexdaylist[1])
        globpersonaltime[2] = int(indexdaylist[2])
        print(globpersonaltime)
        myftp.run()
        timeindex = int(timeindex) + 3600 * 24