Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 113 additions & 0 deletions src/attributecode/attribute.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@

import csv
import json
import warnings
from collections import OrderedDict

import openpyxl

from attributecode import CRITICAL, Error


def read_csv_rows(location):
"""Read CSV rows as lists of values."""
with open(location, encoding="utf-8-sig", errors="replace") as csvfile:
yield from csv.reader(csvfile)


def read_csv_dicts(location):
"""Read CSV rows as dictionaries with normalized column names."""
with open(location, mode="r", encoding="utf-8-sig", errors="replace") as csvfile:
return [
{key.lower().strip(): value for key, value in row.items()}
for row in csv.DictReader(csvfile)
]


def read_json(location):
"""Read and return JSON data without changing its structure."""
with open(location, encoding="utf-8", errors="replace") as jsonfile:
return json.load(jsonfile)


def read_excel(location, worksheet=None):
"""Read an Excel worksheet into a list of ordered dictionaries."""
results = []
errors = []

with warnings.catch_warnings(record=True):
workbook = openpyxl.load_workbook(location)

if worksheet:
if worksheet not in workbook.sheetnames:
raise ValueError(
"The input worksheet name does not exist: {}".format(worksheet)
)
sheet = workbook[worksheet]
else:
sheet = workbook.active

headers = [
sheet.cell(row=1, column=index).value
for index in range(1, sheet.max_column + 1)
]

seen = set()
for header in headers:
if header in seen:
msg = "Duplicated column name, {}, detected.".format(header)
errors.append(Error(CRITICAL, msg))
return errors, results
seen.add(header)

for row in sheet.iter_rows(min_row=2, values_only=True):
row_dict = OrderedDict(
(header, value if value is not None else "")
for header, value in zip(headers, row)
)
results.append(row_dict)

return errors, results


def write_csv(location, data, field_names=None):
"""Write dictionaries to a CSV file or file-like object."""
if field_names is None:
field_names = list(data[0].keys()) if data else []

if hasattr(location, "write"):
output = location
writer = csv.DictWriter(output, fieldnames=field_names)
writer.writeheader()
writer.writerows(data)
else:
with open(
location, "w", encoding="utf-8", newline="", errors="replace"
) as output:
writer = csv.DictWriter(output, fieldnames=field_names)
writer.writeheader()
writer.writerows(data)


def write_json(location, data, indent=2):
"""Write JSON data to a file or file-like object."""
if hasattr(location, "write"):
json.dump(data, location, indent=indent)
else:
with open(location, "w", encoding="utf-8") as output:
json.dump(data, output, indent=indent)


def write_excel(location, data):
"""Write dictionaries to an Excel workbook."""
workbook = openpyxl.Workbook()
sheet = workbook.active

headers = list(data[0].keys()) if data else []
if headers:
sheet.append(headers)

for item in data:
sheet.append([item.get(header) for header in headers])

workbook.save(location)
34 changes: 10 additions & 24 deletions src/attributecode/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,10 @@
from attributecode import saneyaml
from attributecode import gen
from attributecode import util
from attributecode.transform import write_excel
from attributecode.util import add_unc
from attributecode.util import boolean_fields
from attributecode.util import copy_license_notice_files
from attributecode.util import copy_file
from attributecode.util import csv
from attributecode.util import file_fields
from attributecode.util import filter_errors
from attributecode.util import get_spdx_key_and_lic_key_from_licdb
Expand All @@ -65,6 +63,7 @@
from attributecode.util import UNC_PREFIX
from attributecode.util import ungroup_licenses
from attributecode.util import ungroup_licenses_from_sctk
from attributecode import io

genereated_tk_version = "# Generated with AboutCode Toolkit Version %s \n\n" % __version__

Expand Down Expand Up @@ -1915,40 +1914,27 @@ def save_as_json(location, about_dicts):
Save the given data as a JSON file or print it to standard output.
"""
data = util.format_about_dict_for_json_output(about_dicts)
if location == "-":
json.dump(data, sys.stdout, indent=2)
else:
with open(location, mode="w") as output_file:
output_file.write(json.dumps(data, indent=2))
output = sys.stdout if location == "-" else location

io.write_json(output, data, indent=2)


def save_as_csv(location, about_dicts, field_names):
"""
Save the given data as a CSV file or print it to standard output.
"""
if location == "-":
writer = csv.DictWriter(sys.stdout, field_names)
writer.writeheader()
csv_formatted_list = util.format_about_dict_output(about_dicts)
for row in csv_formatted_list:
writer.writerow(row)
else:
with open(
location, mode="w", encoding="utf-8", newline="", errors="replace"
) as output_file:
writer = csv.DictWriter(output_file, field_names)
writer.writeheader()
csv_formatted_list = util.format_about_dict_output(about_dicts)
for row in csv_formatted_list:
writer.writerow(row)
csv_formatted_list = util.format_about_dict_output(about_dicts)
output = sys.stdout if location == "-" else location

io.write_csv(output, csv_formatted_list, field_names)


def save_as_excel(location, about_dicts):
"""
Save the given data as a Excel file.
Save the given data as an Excel file.
"""
formatted_list = util.format_about_dict_output(about_dicts)
write_excel(location, formatted_list)
io.write_excel(location, formatted_list)


def pre_process_and_fetch_license_dict(
Expand Down
122 changes: 16 additions & 106 deletions src/attributecode/transform.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,29 +13,28 @@
# limitations under the License.
# ============================================================================

import json
from collections import Counter, OrderedDict

from collections import Counter
from itertools import zip_longest

import attr
import openpyxl

from attributecode import CRITICAL
from attributecode import Error
from attributecode import saneyaml
from attributecode.util import csv
from attributecode import io
from attributecode.util import replace_tab_with_spaces



def transform_csv(location):
"""
Read a CSV file at `location` and convert data into list of dictionaries.
Read a CSV file and convert data into a list of dictionaries.
"""
errors = []
new_data = []
rows = read_csv_rows(location)
data = iter(rows)
names = next(rows)
rows = iter(io.read_csv_rows(location))
names = next(rows, [])

field_names = strip_trailing_fields_csv(names)
dupes = check_duplicate_fields(field_names)

Expand All @@ -44,37 +43,30 @@ def transform_csv(location):
for name in dupes:
errors.append(Error(CRITICAL, msg % locals()))

new_data = []
if not errors:
# Convert to dicts
new_data = [dict(zip_longest(field_names, item)) for item in data]
new_data = [dict(zip_longest(field_names, item)) for item in rows]

return new_data, errors



def transform_json(location):
"""
Read a JSON file at `location` and convert data into list of dictionaries.
Read a JSON file and convert data into a list of dictionaries.
"""
errors = []
new_data = []
items = read_json(location)
items = io.read_json(location)
data = normalize_dict_data(items)
new_data = strip_trailing_fields_json(data)

return new_data, errors
return new_data, []


def transform_excel(location, worksheet=None):
"""
Read a XLSX file at `location` and convert data into list of dictionaries.
Read an XLSX file and convert data into a list of dictionaries.
"""
errors = []
new_data = []
dupes, new_data = read_excel(location, worksheet)
if dupes:
msg = "Duplicated field name: %(name)s"
for name in dupes:
errors.append(Error(CRITICAL, msg % locals()))
errors, new_data = io.read_excel(location, worksheet)
return new_data, errors


Expand Down Expand Up @@ -360,85 +352,3 @@ def read_csv_rows(location):
for row in reader:
yield row


def read_json(location):
"""
Yield rows (as a list of values) from a CSV file at `location`.
"""
with open(location, encoding="utf-8", errors="replace") as jsonfile:
return json.load(jsonfile)


def write_csv(location, data):
"""
Write a CSV file at `location` with the `data` which is a list of ordered dicts.
"""
field_names = list(data[0].keys())
with open(location, "w", encoding="utf-8", newline="\n", errors="replace") as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=field_names)
writer.writeheader()
writer.writerows(data)


def write_json(location, data):
"""
Write a JSON file at `location` the `data` list of ordered dicts.
"""
with open(location, "w") as jsonfile:
json.dump(data, jsonfile, indent=3)


def read_excel(location, worksheet=None):
"""
Read XLSX at `location`, return a list of ordered dictionaries, one
for each row.
"""
results = []
errors = []
input_bom = openpyxl.load_workbook(location)
if worksheet:
sheet_obj = input_bom[worksheet]
else:
sheet_obj = input_bom.active
max_col = sheet_obj.max_column

index = 1
col_keys = []
mapping_dict = {}
while index <= max_col:
value = sheet_obj.cell(row=1, column=index).value
if value in col_keys:
msg = "Duplicated column name, " + str(value) + ", detected."
errors.append(Error(CRITICAL, msg))
return errors, results
if value in mapping_dict:
value = mapping_dict[value]
col_keys.append(value)
index = index + 1

for row in sheet_obj.iter_rows(min_row=2, values_only=True):
row_dict = OrderedDict()
index = 0
while index < max_col:
value = row[index]
if value:
row_dict[col_keys[index]] = value
else:
row_dict[col_keys[index]] = ""
index = index + 1
results.append(row_dict)
return errors, results


def write_excel(location, data):
wb = openpyxl.Workbook()
ws = wb.active

# Get the header
headers = list(data[0].keys())
ws.append(headers)

for elements in data:
ws.append([elements.get(h) for h in headers])

wb.save(location)
Loading