SAM.gov Data Structure Analysis: The 400+ Line Parsing Nightmare

{ "opportunitiesData": [ { "noticeId": "abc123def456ghi789", "title": "Software Development Services", "sol": "W52P1J-25-R-0001", "fullParentPathName": "Department of Defense.Department of the Army.Army Contracting Command.Army Contracting Command - Detroit Arsenal (ACC-DTA)", "fullParentPathCode": "DOD.DA.ACC.ACC-DTA", "postedDate": "2025-11-01", "type": "Solicitation", "baseType": "o", "archiveType": "auto15", "archiveDate": "2025-12-16", "typeOfSetAsideDescription": "Total Small Business Set-Aside (FAR 19.5)", "typeOfSetAside": "SBA", "responseDeadLine": "2025-11-30T17:00:00-05:00", "pointOfContact": [ { "type": "primary", "title": "", "fullName": "John Smith", "email": "[email protected]", "phone": "586-555-1234", "fax": null }, { "type": "secondary", "title": "Contracting Officer", "fullName": "Jane Doe", "email": "", "phone": "586-555-5678", "fax": null } ], "placeOfPerformance": { "streetAddress": "123 Main Street", "streetAddress2": "Suite 100", "city": { "code": "48397", "name": "Warren" }, "state": { "code": "MI", "name": "Michigan" }, "zip": "48397", "country": { "code": "USA", "name": "UNITED STATES" } }, "organizationType": { "code": "O", "name": "Office" }, "naicsCode": [ { "code": "541511", "title": "Custom Computer Programming Services" }, { "code": "541512", "title": "Computer Systems Design Services" } ], "additionalInfoLink": "https://sam.gov/opp/abc123def456ghi789/view", "uiLink": "https://sam.gov/opp/abc123def456ghi789/view", "links": [ { "rel": "self", "href": "https://api.sam.gov/opportunities/v2/abc123def456ghi789" } ], "resourceLinks": [ { "type": "document", "name": "Amendment 001", "link": "https://sam.gov/api/prod/opps/v3/opportunities/resources/files/abc123/download?&token=..." }, { "type": "document", "name": "Original Solicitation", "link": "https://sam.gov/api/prod/opps/v3/opportunities/resources/files/def456/download?&token=..." } ], "officeAddress": { "zipcode": "48397", "city": "Warren", "countryCode": "USA", "state": "MI" }, // Award information (if available) - completely separate structure "award": { "date": "2025-12-15", "number": "W52P1J-25-C-0001", "amount": 150000, "lineItemNumber": "0001", "awardee": { "name": "ACME Software Solutions", "location": { "streetAddress": "456 Tech Drive", "city": "Detroit", "state": "MI", "zipCode": "48201", "countryCode": "USA" }, "ueiSAM": "ABC123DEF456", "cageCode": "1A2B3" } } } ], "totalRecords": 1247, "offset": 0, "limit": 10 }

import re from datetime import datetime, timezone from typing import Dict, List, Optional, Any class SAMOpportunityParser: """Complex parser for SAM.gov opportunity data""" def __init__(self): # Set-aside code translations self.set_aside_codes = { 'SBA': 'Small Business Set-Aside', 'A6': '8(a) Set-Aside', 'HZC': 'HUBZone Set-Aside', 'SDVOSBC': 'Service-Disabled Veteran-Owned Small Business', 'WOSB': 'Women-Owned Small Business', 'EDWOSB': 'Economically Disadvantaged Women-Owned Small Business', '': 'Full and Open Competition' } # Organization type mappings self.org_type_codes = { 'O': 'Office', 'D': 'Department', 'A': 'Agency', 'S': 'Sub-Agency' } def parse_opportunity(self, raw_data: Dict) -> Dict: """Parse a single opportunity from SAM.gov response""" try: # Extract basic fields with null checking opportunity = { 'notice_id': self._safe_get(raw_data, 'noticeId'), 'title': self._safe_get(raw_data, 'title', '').strip(), 'solicitation_number': self._safe_get(raw_data, 'sol'), 'posted_date': self._parse_date(raw_data.get('postedDate')), 'response_deadline': self._parse_datetime(raw_data.get('responseDeadLine')), 'notice_type': self._safe_get(raw_data, 'type'), 'base_type': self._safe_get(raw_data, 'baseType'), 'archive_date': self._parse_date(raw_data.get('archiveDate')), } # Parse complex agency hierarchy agency_data = self._parse_agency_hierarchy(raw_data) opportunity.update(agency_data) # Parse set-aside information opportunity['set_aside'] = self._parse_set_aside(raw_data) # Parse contact information (complex nested array) opportunity['contacts'] = self._parse_contacts(raw_data.get('pointOfContact', [])) # Parse performance location (deeply nested) opportunity['performance_location'] = self._parse_location( raw_data.get('placeOfPerformance', {}) ) # Parse office address (different structure than performance location) opportunity['office_address'] = self._parse_office_address( raw_data.get('officeAddress', {}) ) # Parse NAICS codes (array of objects) opportunity['naics_codes'] = self._parse_naics_codes( raw_data.get('naicsCode', []) ) # Parse organization type opportunity['organization_type'] = self._parse_organization_type(raw_data) # Parse resource links (documents, amendments) opportunity['resource_links'] = self._parse_resource_links( raw_data.get('resourceLinks', []) ) # Parse award information (if available) opportunity['award_info'] = self._parse_award_info( raw_data.get('award', {}) ) # Generate clean URLs opportunity['sam_url'] = self._generate_sam_url(opportunity['notice_id']) # Extract additional metadata opportunity['total_records'] = raw_data.get('totalRecords') return opportunity except Exception as e: # Robust error handling for malformed data print(f"Error parsing opportunity {raw_data.get('noticeId', 'unknown')}: {e}") return self._create_error_record(raw_data, str(e)) def _safe_get(self, data: Dict, key: str, default: Any = None) -> Any: """Safely extract value with null checking""" value = data.get(key, default) if isinstance(value, str): return value.strip() if value else default return value if value is not None else default def _parse_agency_hierarchy(self, data: Dict) -> Dict: """Parse complex agency hierarchy string""" full_path = data.get('fullParentPathName', '') path_code = data.get('fullParentPathCode', '') # Split hierarchy: "Dept.Agency.Sub-Agency.Office" path_parts = full_path.split('.') code_parts = path_code.split('.') return { 'department': path_parts[0] if len(path_parts) > 0 else '', 'agency': path_parts[1] if len(path_parts) > 1 else '', 'sub_agency': path_parts[2] if len(path_parts) > 2 else '', 'office': path_parts[3] if len(path_parts) > 3 else '', 'department_code': code_parts[0] if len(code_parts) > 0 else '', 'agency_code': code_parts[1] if len(code_parts) > 1 else '', 'full_agency_name': full_path, 'full_agency_code': path_code } def _parse_set_aside(self, data: Dict) -> Dict: """Parse set-aside information with code translation""" code = data.get('typeOfSetAside', '') description = data.get('typeOfSetAsideDescription', '') return { 'code': code, 'description': description, 'standardized_name': self.set_aside_codes.get(code, code), 'is_small_business': code in ['SBA', 'A6', 'HZC', 'SDVOSBC', 'WOSB', 'EDWOSB'] } def _parse_contacts(self, contacts_data: List[Dict]) -> List[Dict]: """Parse contact array with inconsistent structure""" contacts = [] for contact in contacts_data: if not isinstance(contact, dict): continue parsed_contact = { 'type': contact.get('type', '').lower(), 'title': self._safe_get(contact, 'title', ''), 'name': self._safe_get(contact, 'fullName', ''), 'email': self._clean_email(contact.get('email', '')), 'phone': self._clean_phone(contact.get('phone', '')), 'fax': self._clean_phone(contact.get('fax', '')) } # Skip contacts with no useful information if parsed_contact['name'] or parsed_contact['email']: contacts.append(parsed_contact) return contacts def _parse_location(self, location_data: Dict) -> Dict: """Parse complex nested location structure""" if not location_data: return {} # Handle nested city/state/country objects city_obj = location_data.get('city', {}) state_obj = location_data.get('state', {}) country_obj = location_data.get('country', {}) return { 'street_address': self._safe_get(location_data, 'streetAddress', ''), 'street_address_2': self._safe_get(location_data, 'streetAddress2', ''), 'city': city_obj.get('name', '') if isinstance(city_obj, dict) else str(city_obj), 'city_code': city_obj.get('code', '') if isinstance(city_obj, dict) else '', 'state': state_obj.get('code', '') if isinstance(state_obj, dict) else str(state_obj), 'state_name': state_obj.get('name', '') if isinstance(state_obj, dict) else '', 'zip_code': self._safe_get(location_data, 'zip', ''), 'country': country_obj.get('code', '') if isinstance(country_obj, dict) else str(country_obj), 'country_name': country_obj.get('name', '') if isinstance(country_obj, dict) else '' } def _parse_office_address(self, office_data: Dict) -> Dict: """Parse office address (different structure than performance location)""" if not office_data: return {} return { 'city': self._safe_get(office_data, 'city', ''), 'state': self._safe_get(office_data, 'state', ''), 'zip_code': self._safe_get(office_data, 'zipcode', ''), 'country': self._safe_get(office_data, 'countryCode', '') } def _parse_naics_codes(self, naics_data: List[Dict]) -> List[Dict]: """Parse NAICS code array""" naics_codes = [] for naics in naics_data: if not isinstance(naics, dict): continue parsed_naics = { 'code': self._safe_get(naics, 'code', ''), 'title': self._safe_get(naics, 'title', ''), 'is_primary': len(naics_codes) == 0 # First one is primary } if parsed_naics['code']: naics_codes.append(parsed_naics) return naics_codes def _parse_award_info(self, award_data: Dict) -> Optional[Dict]: """Parse award information (completely different structure)""" if not award_data: return None # Parse awardee information (nested in award object) awardee_data = award_data.get('awardee', {}) awardee_location = awardee_data.get('location', {}) return { 'award_date': self._parse_date(award_data.get('date')), 'award_number': self._safe_get(award_data, 'number', ''), 'award_amount': self._parse_amount(award_data.get('amount')), 'line_item_number': self._safe_get(award_data, 'lineItemNumber', ''), 'awardee_name': self._safe_get(awardee_data, 'name', ''), 'awardee_uei': self._safe_get(awardee_data, 'ueiSAM', ''), 'awardee_cage_code': self._safe_get(awardee_data, 'cageCode', ''), 'awardee_address': { 'street': self._safe_get(awardee_location, 'streetAddress', ''), 'city': self._safe_get(awardee_location, 'city', ''), 'state': self._safe_get(awardee_location, 'state', ''), 'zip_code': self._safe_get(awardee_location, 'zipCode', ''), 'country': self._safe_get(awardee_location, 'countryCode', '') } } def _parse_date(self, date_str: Optional[str]) -> Optional[str]: """Parse various date formats from SAM.gov""" if not date_str: return None try: # Handle multiple date formats for fmt in ['%Y-%m-%d', '%m/%d/%Y', '%Y-%m-%dT%H:%M:%S']: try: dt = datetime.strptime(date_str.split('T')[0], fmt) return dt.strftime('%Y-%m-%d') except ValueError: continue return date_str # Return original if parsing fails except Exception: return None def _parse_datetime(self, datetime_str: Optional[str]) -> Optional[str]: """Parse datetime with timezone handling""" if not datetime_str: return None try: # Remove timezone suffix for parsing clean_dt = re.sub(r'[-+]\d{2}:\d{2}$', '', datetime_str) dt = datetime.fromisoformat(clean_dt) return dt.isoformat() except Exception: return datetime_str def _clean_email(self, email: str) -> str: """Clean and validate email addresses""" if not email: return '' email = email.strip().lower() # Basic email validation if '@' in email and '.' in email.split('@')[-1]: return email else: return '' def _clean_phone(self, phone: str) -> str: """Clean phone number format""" if not phone: return '' # Remove non-numeric characters except + clean_phone = re.sub(r'[^\d+\-\(\)\s]', '', phone.strip()) return clean_phone if len(re.sub(r'[^\d]', '', clean_phone)) >= 10 else '' def _parse_amount(self, amount: Any) -> Optional[float]: """Parse monetary amounts""" if amount is None: return None try: if isinstance(amount, (int, float)): return float(amount) elif isinstance(amount, str): # Remove currency symbols and commas clean_amount = re.sub(r'[^\d.]', '', amount) return float(clean_amount) if clean_amount else None except ValueError: return None return None def _generate_sam_url(self, notice_id: str) -> str: """Generate SAM.gov URL for opportunity""" return f"https://sam.gov/opp/{notice_id}/view" if notice_id else "" # ... additional helper methods for resource links, organization types, etc. # This is just a fraction of the total parsing code needed! # Usage example (still complex after 400+ lines of parsing code) def process_sam_response(sam_response: Dict) -> List[Dict]: """Process SAM.gov API response""" parser = SAMOpportunityParser() opportunities = [] for opp_data in sam_response.get('opportunitiesData', []): parsed_opp = parser.parse_opportunity(opp_data) opportunities.append(parsed_opp) return opportunities

{ "data": [ { "notice_id": "abc123def456ghi789", "title": "Software Development Services", "solicitation_number": "W52P1J-25-R-0001", "agency": "Department of Defense", "department": "Department of Defense", "sub_agency": "Army Contracting Command", "office": "ACC - Detroit Arsenal", "posted_date": "2025-11-01", "response_deadline": "2025-11-30T17:00:00-05:00", "notice_type": "Solicitation", "set_aside_type": "Small Business Set-Aside", "set_aside_code": "SBA", "naics": ["541511", "541512"], "naics_titles": ["Custom Computer Programming Services", "Computer Systems Design Services"], "primary_naics": "541511", "contact_name": "John Smith", "contact_email": "[email protected]", "contact_phone": "586-555-1234", "secondary_contact": "Jane Doe", "secondary_email": "", "secondary_phone": "586-555-5678", "performance_city": "Warren", "performance_state": "MI", "performance_state_name": "Michigan", "performance_zip": "48397", "performance_country": "USA", "performance_address": "123 Main Street, Suite 100", "sam_url": "https://sam.gov/opp/abc123def456ghi789/view", "description_text": "The Army requires software development services for...", "award_date": "2025-12-15", "award_number": "W52P1J-25-C-0001", "award_amount": 150000.00, "awardee_name": "ACME Software Solutions", "awardee_location": "Detroit, MI", "awardee_uei": "ABC123DEF456", "archive_date": "2025-12-16", "last_updated": "2025-11-01T10:30:00Z", "active": true } ], "pagination": { "total": 1247, "limit": 100, "offset": 0, "has_next": true } }

import requests # Get clean, parsed data instantly response = requests.get( 'https://govconapi.com/api/v1/opportunities/search', headers={'Authorization': 'Bearer your_api_key'}, params={'naics': '541511', 'limit': 100} ) opportunities = response.json()['data'] # Process clean data directly - no parsing needed! for opp in opportunities: print(f"Title: {opp['title']}") print(f"Agency: {opp['agency']}") # Clean, not nested print(f"Contact: {opp['contact_email']}") # Direct access print(f"Description: {opp['description_text']}") # Included! print(f"Award Amount: ${opp['award_amount'] or 'TBD'}") # Integrated print("---") # That's it! No parsing complexity, no error handling, no data normalization.