Skip to content

GCP Cloud Storage Connector

GcpCloudStorage

Instantiate a GCS connector. Args: credential_file (str): Credential json file proxy (str): Proxy address

Source code in honeydew/gcp_storage.py
Python
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
class GcpCloudStorage:
    """
    Instantiate a GCS connector.
    Args:
        credential_file (str): Credential json file
        proxy (str): Proxy address
    """
    def __init__(self, credential_file, proxy=''):
        self.credential_file = credential_file
        self.proxy = proxy
        os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = credential_file
        if proxy != '':
            os.environ['HTTP_PROXY'] = proxy
            os.environ['HTTPS_PROXY'] = proxy

    def download_blob(self, project_id, bucket_id, source_blob_path, destination_path):
        """
        Download a single object from Google Cloud Storage (GCS).

        Args:
            project_id (str): Project ID
            bucket_id (str): Bucket ID
            source_blob_path (str): The path of source object. Example: 'gcs-directory/my-filename.txt'
            destination_path (str): Local destination path. Example: '/my-directory/my-filename.txt'

        Returns:
            result (result): Iterator of row data. Reference: https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.job.QueryJob.html?highlight=job%20result#google.cloud.bigquery.job.QueryJob.result
        """
        gcs_client = storage.Client(project=project_id)
        bucket = gcs_client.bucket(bucket_id)
        blob = bucket.blob(source_blob_path)
        blob.download_to_filename(destination_path)
        results = 'OK'
        return results    

    def download_many_blobs(self, project_id, bucket_id, blob_prefix, destination_dir_path, printout=True):
        """
        Download multiple objects which have same prefix pattern from Google Cloud Storage (GCS).

        Args:
            project_id (str): Project ID
            bucket_id (str): Bucket ID
            blob_prefix (str): The blob prefix pattern that wil be downloaded. Example: 'gcs-directory/tickets-20220101-'
            destination_dir_path (str): Local destination directory path. Example: '/my-directory'
            printout (boolean): File name will be displayed if this value is true. Default: True

        Returns:
            result (str): It returns 'OK' when successful            
        """

        delimiter='/'
        storage_client = storage.Client(project_id)
        bucket=storage_client.get_bucket(bucket_id)
        # List blobs iterate in folder 
        blobs=bucket.list_blobs(prefix=blob_prefix, delimiter=delimiter) # Excluding folder inside bucket
        for blob in blobs:
            if printout == True:
                print(blob.name)
            filename_raw = blob.name.split('/')
            filename = filename_raw[len(filename_raw)-1]
            destination_uri = '{}{}'.format(destination_dir_path, filename) 
            blob.download_to_filename(destination_uri)
        results = 'OK'   
        return results

    def upload_blob(self, project_id, bucket_id, local_file, destination_blob):
        """
        Upload a single object from Google Cloud Storage (GCS).

        Args:
            project_id (str): Project ID
            bucket_id (str): Bucket ID
            local_file (str): Local file as source. Example: '/local-directory/my-filename.txt'
            destination_blob (str): Destination blob in GCS bucket. Example: 'gcs-directory/my-filename.txt'

        Returns:
            result (str): It returns 'OK' when successful
        """
        gcs_client = storage.Client(project=project_id)
        bucket = gcs_client.bucket(bucket_id)
        blob = bucket.blob(destination_blob)
        blob.upload_from_filename(local_file)
        results = 'OK'
        return results          

    def upload_many_blobs(self, project_id, bucket_id, local_dir, destination_dir):
        """
        Upload multiple objects to Google Cloud Storage (GCS).

        Args:
            project_id (str): Project ID
            bucket_id (str): Bucket ID
            local_dir (str): Local directory as source. Example: '/local-directory'
            destination_dir (str): Destination directory in GCS bucket. Example: 'gcs-directory'

        Returns:
            result (str): It returns 'OK' when successful
        """
        gcs_client = storage.Client(project=project_id)
        bucket = gcs_client.bucket(bucket_id)
        for filename in os.listdir(local_dir):
            blob = bucket.blob(f'{destination_dir}/{filename}')
            blob.upload_from_filename(f'{local_dir}/{filename}')
        results = 'OK'
        return results

    def delete_blob(self, project_id, bucket_id, blob_name):
        """
        Delete a single object from Google Cloud Storage (GCS).

        Args:
            project_id (str): Project ID
            bucket_id (str): Bucket ID
            blob_name (str): Blob name. Example: 'gcs-directory/my-filename.txt'

        Returns:
            result (str): It returns 'OK' when successful
        """
        gcs_client = storage.Client(project=project_id)
        bucket = gcs_client.bucket(bucket_id)
        blob = bucket.blob(blob_name)
        blob.delete()
        results = 'OK'
        return results

    def delete_many_blobs(self, project_id, bucket_id, blob_prefix):
        """
        Delete multiple objects from Google Cloud Storage (GCS).

        Args:
            project_id (str): Project ID
            bucket_id (str): Bucket ID
            blob_prefix (str): Blob prefix. Example: 'gcs-directory/my-filename.txt'

        Returns:
            result (str): It returns 'OK' when successful
        """
        delimiter='/'
        storage_client = storage.Client(project_id)
        bucket=storage_client.get_bucket(bucket_id)
        # List blobs iterate in folder 
        blobs=bucket.list_blobs(prefix=blob_prefix, delimiter=delimiter)
        for blob in blobs:
            blob.delete()
        results = 'OK'
        return results

    def list_blobs(self, project_id, bucket_id, blob_prefix):
        """
        List objects from Google Cloud Storage (GCS).

        Args:
            project_id (str): Project ID
            bucket_id (str): Bucket ID
            blob_prefix (str): Blob prefix. Example: 'gcs-directory/my-filename.txt'

        Returns:
            blob_list (list): Blob list
        """
        delimiter='/'
        storage_client = storage.Client(project_id)
        bucket=storage_client.get_bucket(bucket_id)
        # List blobs iterate in folder 
        blobs=bucket.list_blobs(prefix=blob_prefix, delimiter=delimiter)
        blob_list = []
        for blob in blobs:
            blob_list.append(blob.name)
        results = 'OK'
        return blob_list

    def copy_blob_between_buckets(self, project_id, source_bucket_id, destination_bucket_id, blob_name):
        """
        Copy a single object between buckets in Google Cloud Storage (GCS).

        Args:
            project_id (str): Project ID
            source_bucket_id (str): Source bucket ID
            destination_bucket_id (str): Destination bucket ID
            blob_name (str): Blob name. Example: 'gcs-directory/my-filename.txt'

        Returns:
            result (str): It returns 'OK' when successful
        """
        gcs_client = storage.Client(project=project_id)
        source_bucket = gcs_client.bucket(source_bucket_id)
        destination_bucket = gcs_client.bucket(destination_bucket_id)
        blob = source_bucket.blob(blob_name)
        new_blob = source_bucket.copy_blob(blob, destination_bucket, blob_name)
        results = 'OK'
        return results

    def copy_many_blobs_between_buckets(self, project_id, source_bucket_id, destination_bucket_id, blob_prefix):
        """
        Copy multiple objects between buckets in Google Cloud Storage (GCS).

        Args:
            project_id (str): Project ID
            source_bucket_id (str): Source bucket ID
            destination_bucket_id (str): Destination bucket ID
            blob_prefix (str): Blob prefix. Example: 'gcs-directory/my-filename.txt'

        Returns:
            result (str): It returns 'OK' when successful
        """
        delimiter='/'
        storage_client = storage.Client(project_id)
        source_bucket = storage_client.bucket(source_bucket_id)
        destination_bucket = storage_client.bucket(destination_bucket_id)
        # List blobs iterate in folder 
        blobs=source_bucket.list_blobs(prefix=blob_prefix, delimiter=delimiter)
        for blob in blobs:
            new_blob = source_bucket.copy_blob(blob, destination_bucket, blob.name)
        results = 'OK'
        return results

copy_blob_between_buckets(project_id, source_bucket_id, destination_bucket_id, blob_name)

Copy a single object between buckets in Google Cloud Storage (GCS).

Parameters:

Name Type Description Default
project_id str

Project ID

 required
source_bucket_id str

Source bucket ID

 required
destination_bucket_id str

Destination bucket ID

 required
blob_name str

Blob name. Example: 'gcs-directory/my-filename.txt'

 required

Returns:

Name Type Description
result str

It returns 'OK' when successful
Source code in honeydew/gcp_storage.py
Python
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
def copy_blob_between_buckets(self, project_id, source_bucket_id, destination_bucket_id, blob_name):
    """
    Copy a single object between buckets in Google Cloud Storage (GCS).

    Args:
        project_id (str): Project ID
        source_bucket_id (str): Source bucket ID
        destination_bucket_id (str): Destination bucket ID
        blob_name (str): Blob name. Example: 'gcs-directory/my-filename.txt'

    Returns:
        result (str): It returns 'OK' when successful
    """
    gcs_client = storage.Client(project=project_id)
    source_bucket = gcs_client.bucket(source_bucket_id)
    destination_bucket = gcs_client.bucket(destination_bucket_id)
    blob = source_bucket.blob(blob_name)
    new_blob = source_bucket.copy_blob(blob, destination_bucket, blob_name)
    results = 'OK'
    return results

copy_many_blobs_between_buckets(project_id, source_bucket_id, destination_bucket_id, blob_prefix)

Copy multiple objects between buckets in Google Cloud Storage (GCS).

Parameters:

Name Type Description Default
project_id str

Project ID

 required
source_bucket_id str

Source bucket ID

 required
destination_bucket_id str

Destination bucket ID

 required
blob_prefix str

Blob prefix. Example: 'gcs-directory/my-filename.txt'

 required

Returns:

Name Type Description
result str

It returns 'OK' when successful
Source code in honeydew/gcp_storage.py
Python
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
def copy_many_blobs_between_buckets(self, project_id, source_bucket_id, destination_bucket_id, blob_prefix):
    """
    Copy multiple objects between buckets in Google Cloud Storage (GCS).

    Args:
        project_id (str): Project ID
        source_bucket_id (str): Source bucket ID
        destination_bucket_id (str): Destination bucket ID
        blob_prefix (str): Blob prefix. Example: 'gcs-directory/my-filename.txt'

    Returns:
        result (str): It returns 'OK' when successful
    """
    delimiter='/'
    storage_client = storage.Client(project_id)
    source_bucket = storage_client.bucket(source_bucket_id)
    destination_bucket = storage_client.bucket(destination_bucket_id)
    # List blobs iterate in folder 
    blobs=source_bucket.list_blobs(prefix=blob_prefix, delimiter=delimiter)
    for blob in blobs:
        new_blob = source_bucket.copy_blob(blob, destination_bucket, blob.name)
    results = 'OK'
    return results

delete_blob(project_id, bucket_id, blob_name)

Delete a single object from Google Cloud Storage (GCS).

Parameters:

Name Type Description Default
project_id str

Project ID

 required
bucket_id str

Bucket ID

 required
blob_name str

Blob name. Example: 'gcs-directory/my-filename.txt'

 required

Returns:

Name Type Description
result str

It returns 'OK' when successful
Source code in honeydew/gcp_storage.py
Python
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
def delete_blob(self, project_id, bucket_id, blob_name):
    """
    Delete a single object from Google Cloud Storage (GCS).

    Args:
        project_id (str): Project ID
        bucket_id (str): Bucket ID
        blob_name (str): Blob name. Example: 'gcs-directory/my-filename.txt'

    Returns:
        result (str): It returns 'OK' when successful
    """
    gcs_client = storage.Client(project=project_id)
    bucket = gcs_client.bucket(bucket_id)
    blob = bucket.blob(blob_name)
    blob.delete()
    results = 'OK'
    return results

delete_many_blobs(project_id, bucket_id, blob_prefix)

Delete multiple objects from Google Cloud Storage (GCS).

Parameters:

Name Type Description Default
project_id str

Project ID

 required
bucket_id str

Bucket ID

 required
blob_prefix str

Blob prefix. Example: 'gcs-directory/my-filename.txt'

 required

Returns:

Name Type Description
result str

It returns 'OK' when successful
Source code in honeydew/gcp_storage.py
Python
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
def delete_many_blobs(self, project_id, bucket_id, blob_prefix):
    """
    Delete multiple objects from Google Cloud Storage (GCS).

    Args:
        project_id (str): Project ID
        bucket_id (str): Bucket ID
        blob_prefix (str): Blob prefix. Example: 'gcs-directory/my-filename.txt'

    Returns:
        result (str): It returns 'OK' when successful
    """
    delimiter='/'
    storage_client = storage.Client(project_id)
    bucket=storage_client.get_bucket(bucket_id)
    # List blobs iterate in folder 
    blobs=bucket.list_blobs(prefix=blob_prefix, delimiter=delimiter)
    for blob in blobs:
        blob.delete()
    results = 'OK'
    return results

download_blob(project_id, bucket_id, source_blob_path, destination_path)

Download a single object from Google Cloud Storage (GCS).

Parameters:

Name Type Description Default
project_id str

Project ID

 required
bucket_id str

Bucket ID

 required
source_blob_path str

The path of source object. Example: 'gcs-directory/my-filename.txt'

 required
destination_path str

Local destination path. Example: '/my-directory/my-filename.txt'

 required

Returns:

Name Type Description
result result

Iterator of row data. Reference: https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.job.QueryJob.html?highlight=job%20result#google.cloud.bigquery.job.QueryJob.result
Source code in honeydew/gcp_storage.py
Python
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
def download_blob(self, project_id, bucket_id, source_blob_path, destination_path):
    """
    Download a single object from Google Cloud Storage (GCS).

    Args:
        project_id (str): Project ID
        bucket_id (str): Bucket ID
        source_blob_path (str): The path of source object. Example: 'gcs-directory/my-filename.txt'
        destination_path (str): Local destination path. Example: '/my-directory/my-filename.txt'

    Returns:
        result (result): Iterator of row data. Reference: https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.job.QueryJob.html?highlight=job%20result#google.cloud.bigquery.job.QueryJob.result
    """
    gcs_client = storage.Client(project=project_id)
    bucket = gcs_client.bucket(bucket_id)
    blob = bucket.blob(source_blob_path)
    blob.download_to_filename(destination_path)
    results = 'OK'
    return results

download_many_blobs(project_id, bucket_id, blob_prefix, destination_dir_path, printout=True)

Download multiple objects which have same prefix pattern from Google Cloud Storage (GCS).

Parameters:

Name Type Description Default
project_id str

Project ID

 required
bucket_id str

Bucket ID

 required
blob_prefix str

The blob prefix pattern that wil be downloaded. Example: 'gcs-directory/tickets-20220101-'

 required
destination_dir_path str

Local destination directory path. Example: '/my-directory'

 required
printout boolean

File name will be displayed if this value is true. Default: True

 True

Returns:

Name Type Description
result str

It returns 'OK' when successful
Source code in honeydew/gcp_storage.py
Python
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
def download_many_blobs(self, project_id, bucket_id, blob_prefix, destination_dir_path, printout=True):
    """
    Download multiple objects which have same prefix pattern from Google Cloud Storage (GCS).

    Args:
        project_id (str): Project ID
        bucket_id (str): Bucket ID
        blob_prefix (str): The blob prefix pattern that wil be downloaded. Example: 'gcs-directory/tickets-20220101-'
        destination_dir_path (str): Local destination directory path. Example: '/my-directory'
        printout (boolean): File name will be displayed if this value is true. Default: True

    Returns:
        result (str): It returns 'OK' when successful            
    """

    delimiter='/'
    storage_client = storage.Client(project_id)
    bucket=storage_client.get_bucket(bucket_id)
    # List blobs iterate in folder 
    blobs=bucket.list_blobs(prefix=blob_prefix, delimiter=delimiter) # Excluding folder inside bucket
    for blob in blobs:
        if printout == True:
            print(blob.name)
        filename_raw = blob.name.split('/')
        filename = filename_raw[len(filename_raw)-1]
        destination_uri = '{}{}'.format(destination_dir_path, filename) 
        blob.download_to_filename(destination_uri)
    results = 'OK'   
    return results

list_blobs(project_id, bucket_id, blob_prefix)

List objects from Google Cloud Storage (GCS).

Parameters:

Name Type Description Default
project_id str

Project ID

 required
bucket_id str

Bucket ID

 required
blob_prefix str

Blob prefix. Example: 'gcs-directory/my-filename.txt'

 required

Returns:

Name Type Description
blob_list list

Blob list
Source code in honeydew/gcp_storage.py
Python
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
def list_blobs(self, project_id, bucket_id, blob_prefix):
    """
    List objects from Google Cloud Storage (GCS).

    Args:
        project_id (str): Project ID
        bucket_id (str): Bucket ID
        blob_prefix (str): Blob prefix. Example: 'gcs-directory/my-filename.txt'

    Returns:
        blob_list (list): Blob list
    """
    delimiter='/'
    storage_client = storage.Client(project_id)
    bucket=storage_client.get_bucket(bucket_id)
    # List blobs iterate in folder 
    blobs=bucket.list_blobs(prefix=blob_prefix, delimiter=delimiter)
    blob_list = []
    for blob in blobs:
        blob_list.append(blob.name)
    results = 'OK'
    return blob_list

upload_blob(project_id, bucket_id, local_file, destination_blob)

Upload a single object from Google Cloud Storage (GCS).

Parameters:

Name Type Description Default
project_id str

Project ID

 required
bucket_id str

Bucket ID

 required
local_file str

Local file as source. Example: '/local-directory/my-filename.txt'

 required
destination_blob str

Destination blob in GCS bucket. Example: 'gcs-directory/my-filename.txt'

 required

Returns:

Name Type Description
result str

It returns 'OK' when successful
Source code in honeydew/gcp_storage.py
Python
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
def upload_blob(self, project_id, bucket_id, local_file, destination_blob):
    """
    Upload a single object from Google Cloud Storage (GCS).

    Args:
        project_id (str): Project ID
        bucket_id (str): Bucket ID
        local_file (str): Local file as source. Example: '/local-directory/my-filename.txt'
        destination_blob (str): Destination blob in GCS bucket. Example: 'gcs-directory/my-filename.txt'

    Returns:
        result (str): It returns 'OK' when successful
    """
    gcs_client = storage.Client(project=project_id)
    bucket = gcs_client.bucket(bucket_id)
    blob = bucket.blob(destination_blob)
    blob.upload_from_filename(local_file)
    results = 'OK'
    return results

upload_many_blobs(project_id, bucket_id, local_dir, destination_dir)

Upload multiple objects to Google Cloud Storage (GCS).

Parameters:

Name Type Description Default
project_id str

Project ID

 required
bucket_id str

Bucket ID

 required
local_dir str

Local directory as source. Example: '/local-directory'

 required
destination_dir str

Destination directory in GCS bucket. Example: 'gcs-directory'

 required

Returns:

Name Type Description
result str

It returns 'OK' when successful
Source code in honeydew/gcp_storage.py
Python
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
def upload_many_blobs(self, project_id, bucket_id, local_dir, destination_dir):
    """
    Upload multiple objects to Google Cloud Storage (GCS).

    Args:
        project_id (str): Project ID
        bucket_id (str): Bucket ID
        local_dir (str): Local directory as source. Example: '/local-directory'
        destination_dir (str): Destination directory in GCS bucket. Example: 'gcs-directory'

    Returns:
        result (str): It returns 'OK' when successful
    """
    gcs_client = storage.Client(project=project_id)
    bucket = gcs_client.bucket(bucket_id)
    for filename in os.listdir(local_dir):
        blob = bucket.blob(f'{destination_dir}/{filename}')
        blob.upload_from_filename(f'{local_dir}/{filename}')
    results = 'OK'
    return results