From 3e544d1d74507a3d1ebb099425529d829f153f3b Mon Sep 17 00:00:00 2001 From: kriishgandhi98-maker Date: Sun, 22 Nov 2026 16:38:03 +0530 Subject: [PATCH] Add attribute code support --- src/attributecode/attribute.py | 113 ++++++++++++++++++++++++++++++ src/attributecode/model.py | 34 +++------ src/attributecode/transform.py | 122 +++++---------------------------- src/attributecode/util.py | 83 ++++------------------ 4 files changed, 153 insertions(+), 199 deletions(-) create mode 100644 src/attributecode/attribute.py diff --git a/src/attributecode/attribute.py b/src/attributecode/attribute.py new file mode 100644 index 00000000..b70da0e8 --- /dev/null +++ b/src/attributecode/attribute.py @@ -0,0 +1,113 @@ + +import csv +import json +import warnings +from collections import OrderedDict + +import openpyxl + +from attributecode import CRITICAL, Error + + +def read_csv_rows(location): + """Read CSV rows as lists of values.""" + with open(location, encoding="utf-8-sig", errors="replace") as csvfile: + yield from csv.reader(csvfile) + + +def read_csv_dicts(location): + """Read CSV rows as dictionaries with normalized column names.""" + with open(location, mode="r", encoding="utf-8-sig", errors="replace") as csvfile: + return [ + {key.lower().strip(): value for key, value in row.items()} + for row in csv.DictReader(csvfile) + ] + + +def read_json(location): + """Read and return JSON data without changing its structure.""" + with open(location, encoding="utf-8", errors="replace") as jsonfile: + return json.load(jsonfile) + + +def read_excel(location, worksheet=None): + """Read an Excel worksheet into a list of ordered dictionaries.""" + results = [] + errors = [] + + with warnings.catch_warnings(record=True): + workbook = openpyxl.load_workbook(location) + + if worksheet: + if worksheet not in workbook.sheetnames: + raise ValueError( + "The input worksheet name does not exist: {}".format(worksheet) + ) + sheet = workbook[worksheet] + else: + sheet = workbook.active + + headers = [ + sheet.cell(row=1, column=index).value + for index in range(1, sheet.max_column + 1) + ] + + seen = set() + for header in headers: + if header in seen: + msg = "Duplicated column name, {}, detected.".format(header) + errors.append(Error(CRITICAL, msg)) + return errors, results + seen.add(header) + + for row in sheet.iter_rows(min_row=2, values_only=True): + row_dict = OrderedDict( + (header, value if value is not None else "") + for header, value in zip(headers, row) + ) + results.append(row_dict) + + return errors, results + + +def write_csv(location, data, field_names=None): + """Write dictionaries to a CSV file or file-like object.""" + if field_names is None: + field_names = list(data[0].keys()) if data else [] + + if hasattr(location, "write"): + output = location + writer = csv.DictWriter(output, fieldnames=field_names) + writer.writeheader() + writer.writerows(data) + else: + with open( + location, "w", encoding="utf-8", newline="", errors="replace" + ) as output: + writer = csv.DictWriter(output, fieldnames=field_names) + writer.writeheader() + writer.writerows(data) + + +def write_json(location, data, indent=2): + """Write JSON data to a file or file-like object.""" + if hasattr(location, "write"): + json.dump(data, location, indent=indent) + else: + with open(location, "w", encoding="utf-8") as output: + json.dump(data, output, indent=indent) + + +def write_excel(location, data): + """Write dictionaries to an Excel workbook.""" + workbook = openpyxl.Workbook() + sheet = workbook.active + + headers = list(data[0].keys()) if data else [] + if headers: + sheet.append(headers) + + for item in data: + sheet.append([item.get(header) for header in headers]) + + workbook.save(location) diff --git a/src/attributecode/model.py b/src/attributecode/model.py index 66c19656..37124799 100644 --- a/src/attributecode/model.py +++ b/src/attributecode/model.py @@ -48,12 +48,10 @@ from attributecode import saneyaml from attributecode import gen from attributecode import util -from attributecode.transform import write_excel from attributecode.util import add_unc from attributecode.util import boolean_fields from attributecode.util import copy_license_notice_files from attributecode.util import copy_file -from attributecode.util import csv from attributecode.util import file_fields from attributecode.util import filter_errors from attributecode.util import get_spdx_key_and_lic_key_from_licdb @@ -65,6 +63,7 @@ from attributecode.util import UNC_PREFIX from attributecode.util import ungroup_licenses from attributecode.util import ungroup_licenses_from_sctk +from attributecode import io genereated_tk_version = "# Generated with AboutCode Toolkit Version %s \n\n" % __version__ @@ -1915,40 +1914,27 @@ def save_as_json(location, about_dicts): Save the given data as a JSON file or print it to standard output. """ data = util.format_about_dict_for_json_output(about_dicts) - if location == "-": - json.dump(data, sys.stdout, indent=2) - else: - with open(location, mode="w") as output_file: - output_file.write(json.dumps(data, indent=2)) + output = sys.stdout if location == "-" else location + + io.write_json(output, data, indent=2) def save_as_csv(location, about_dicts, field_names): """ Save the given data as a CSV file or print it to standard output. """ - if location == "-": - writer = csv.DictWriter(sys.stdout, field_names) - writer.writeheader() - csv_formatted_list = util.format_about_dict_output(about_dicts) - for row in csv_formatted_list: - writer.writerow(row) - else: - with open( - location, mode="w", encoding="utf-8", newline="", errors="replace" - ) as output_file: - writer = csv.DictWriter(output_file, field_names) - writer.writeheader() - csv_formatted_list = util.format_about_dict_output(about_dicts) - for row in csv_formatted_list: - writer.writerow(row) + csv_formatted_list = util.format_about_dict_output(about_dicts) + output = sys.stdout if location == "-" else location + + io.write_csv(output, csv_formatted_list, field_names) def save_as_excel(location, about_dicts): """ - Save the given data as a Excel file. + Save the given data as an Excel file. """ formatted_list = util.format_about_dict_output(about_dicts) - write_excel(location, formatted_list) + io.write_excel(location, formatted_list) def pre_process_and_fetch_license_dict( diff --git a/src/attributecode/transform.py b/src/attributecode/transform.py index f0972f71..a3860bf2 100644 --- a/src/attributecode/transform.py +++ b/src/attributecode/transform.py @@ -13,29 +13,28 @@ # limitations under the License. # ============================================================================ -import json -from collections import Counter, OrderedDict + +from collections import Counter from itertools import zip_longest import attr -import openpyxl from attributecode import CRITICAL from attributecode import Error from attributecode import saneyaml -from attributecode.util import csv +from attributecode import io from attributecode.util import replace_tab_with_spaces + def transform_csv(location): """ - Read a CSV file at `location` and convert data into list of dictionaries. + Read a CSV file and convert data into a list of dictionaries. """ errors = [] - new_data = [] - rows = read_csv_rows(location) - data = iter(rows) - names = next(rows) + rows = iter(io.read_csv_rows(location)) + names = next(rows, []) + field_names = strip_trailing_fields_csv(names) dupes = check_duplicate_fields(field_names) @@ -44,37 +43,30 @@ def transform_csv(location): for name in dupes: errors.append(Error(CRITICAL, msg % locals())) + new_data = [] if not errors: - # Convert to dicts - new_data = [dict(zip_longest(field_names, item)) for item in data] + new_data = [dict(zip_longest(field_names, item)) for item in rows] return new_data, errors + def transform_json(location): """ - Read a JSON file at `location` and convert data into list of dictionaries. + Read a JSON file and convert data into a list of dictionaries. """ - errors = [] - new_data = [] - items = read_json(location) + items = io.read_json(location) data = normalize_dict_data(items) new_data = strip_trailing_fields_json(data) - return new_data, errors + return new_data, [] def transform_excel(location, worksheet=None): """ - Read a XLSX file at `location` and convert data into list of dictionaries. + Read an XLSX file and convert data into a list of dictionaries. """ - errors = [] - new_data = [] - dupes, new_data = read_excel(location, worksheet) - if dupes: - msg = "Duplicated field name: %(name)s" - for name in dupes: - errors.append(Error(CRITICAL, msg % locals())) + errors, new_data = io.read_excel(location, worksheet) return new_data, errors @@ -360,85 +352,3 @@ def read_csv_rows(location): for row in reader: yield row - -def read_json(location): - """ - Yield rows (as a list of values) from a CSV file at `location`. - """ - with open(location, encoding="utf-8", errors="replace") as jsonfile: - return json.load(jsonfile) - - -def write_csv(location, data): - """ - Write a CSV file at `location` with the `data` which is a list of ordered dicts. - """ - field_names = list(data[0].keys()) - with open(location, "w", encoding="utf-8", newline="\n", errors="replace") as csvfile: - writer = csv.DictWriter(csvfile, fieldnames=field_names) - writer.writeheader() - writer.writerows(data) - - -def write_json(location, data): - """ - Write a JSON file at `location` the `data` list of ordered dicts. - """ - with open(location, "w") as jsonfile: - json.dump(data, jsonfile, indent=3) - - -def read_excel(location, worksheet=None): - """ - Read XLSX at `location`, return a list of ordered dictionaries, one - for each row. - """ - results = [] - errors = [] - input_bom = openpyxl.load_workbook(location) - if worksheet: - sheet_obj = input_bom[worksheet] - else: - sheet_obj = input_bom.active - max_col = sheet_obj.max_column - - index = 1 - col_keys = [] - mapping_dict = {} - while index <= max_col: - value = sheet_obj.cell(row=1, column=index).value - if value in col_keys: - msg = "Duplicated column name, " + str(value) + ", detected." - errors.append(Error(CRITICAL, msg)) - return errors, results - if value in mapping_dict: - value = mapping_dict[value] - col_keys.append(value) - index = index + 1 - - for row in sheet_obj.iter_rows(min_row=2, values_only=True): - row_dict = OrderedDict() - index = 0 - while index < max_col: - value = row[index] - if value: - row_dict[col_keys[index]] = value - else: - row_dict[col_keys[index]] = "" - index = index + 1 - results.append(row_dict) - return errors, results - - -def write_excel(location, data): - wb = openpyxl.Workbook() - ws = wb.active - - # Get the header - headers = list(data[0].keys()) - ws.append(headers) - - for elements in data: - ws.append([elements.get(h) for h in headers]) - - wb.save(location) diff --git a/src/attributecode/util.py b/src/attributecode/util.py index d63e6b6a..16998f52 100644 --- a/src/attributecode/util.py +++ b/src/attributecode/util.py @@ -19,7 +19,6 @@ import csv import json import ntpath -import openpyxl import os import posixpath import re @@ -336,27 +335,18 @@ def resource_name(path): return right.strip() + def load_csv(location): - """ - Read CSV at `location`, return a list of ordered dictionaries, one - for each row. - """ - results = [] - with open(location, mode="r", encoding="utf-8-sig", errors="replace") as csvfile: - for row in csv.DictReader(csvfile): - # convert all the column keys to lower case - updated_row = {key.lower().strip(): value for key, value in row.items()} - results.append(updated_row) - return results + """Read CSV and return rows as dictionaries.""" + from attributecode.io import read_csv_dicts + return read_csv_dicts(location) def load_json(location): - """ - Read JSON file at `location` and return a list of ordered dicts, one for - each entry. - """ - with open(location) as json_file: - results = json.load(json_file) + """Read JSON and return a list of dictionaries.""" + from attributecode.io import read_json + + results = read_json(location) if not isinstance(results, list): results = [results] @@ -772,58 +762,13 @@ def load_scancode_json(location): return updated_results + def load_excel(location, worksheet=None): - """ - Read XLSX at `location`, return a list of ordered dictionaries, one - for each row. - """ - results = [] - errors = [] - import warnings - - # This is to prevent showing the: warn("Workbook contains no default style, apply openpyxl's default") - with warnings.catch_warnings(record=True): - input_bom = openpyxl.load_workbook(location) - sheetnames = input_bom.sheetnames - if worksheet: - if worksheet not in sheetnames: - import sys - - print("The input worksheet name does not exist. Exiting.") - sys.exit(1) - sheet_obj = input_bom[worksheet] - else: - sheet_obj = input_bom.active - print("Working on the " + sheet_obj.title + " worksheet.") - max_col = sheet_obj.max_column - - index = 1 - col_keys = [] - mapping_dict = {} - - while index <= max_col: - value = sheet_obj.cell(row=1, column=index).value - if value in col_keys: - msg = "Duplicated column name, " + str(value) + ", detected." - errors.append(Error(CRITICAL, msg)) - return errors, results - if value in mapping_dict: - value = mapping_dict[value] - col_keys.append(value) - index = index + 1 - - for row in sheet_obj.iter_rows(min_row=2, values_only=True): - row_dict = OrderedDict() - index = 0 - while index < max_col: - value = row[index] - if value: - row_dict[col_keys[index]] = value - else: - row_dict[col_keys[index]] = "" - index = index + 1 - results.append(row_dict) - return errors, results + """Read XLSX and return errors and rows.""" + from attributecode.io import read_excel + + return read_excel(location, worksheet) + def write_licenses(lic_dict, location):